summaryrefslogtreecommitdiff
path: root/ext
diff options
context:
space:
mode:
authorHans-Christoph Steiner <hans@eds.org>2012-09-20 18:34:38 -0400
committerHans-Christoph Steiner <hans@eds.org>2012-09-20 18:34:38 -0400
commit487e15dc239ccdb3344d1c99ce120e872bab4a74 (patch)
treec986d492f6092ca7b4401d91515f74daed17fae2 /ext
parent7bb481fda9ecb134804b49c2ce77ca28f7eea583 (diff)
Imported Upstream version 2.0.6
Diffstat (limited to 'ext')
-rw-r--r--ext/async/sqlite3async.c11
-rw-r--r--ext/fts3/fts3.c389
-rw-r--r--ext/fts3/fts3Int.h58
-rw-r--r--ext/fts3/fts3_aux.c6
-rw-r--r--ext/fts3/fts3_expr.c43
-rw-r--r--ext/fts3/fts3_icu.c5
-rw-r--r--ext/fts3/fts3_porter.c3
-rw-r--r--ext/fts3/fts3_snippet.c47
-rw-r--r--ext/fts3/fts3_term.c16
-rw-r--r--ext/fts3/fts3_test.c213
-rw-r--r--ext/fts3/fts3_tokenizer.c3
-rw-r--r--ext/fts3/fts3_tokenizer.h11
-rw-r--r--ext/fts3/fts3_tokenizer1.c1
-rw-r--r--ext/fts3/fts3_write.c2217
-rw-r--r--ext/fts3/tool/fts3view.c874
-rw-r--r--ext/rtree/rtree.c256
-rw-r--r--ext/rtree/rtree1.test88
-rw-r--r--ext/rtree/rtree4.test43
-rw-r--r--ext/rtree/rtree5.test8
-rw-r--r--ext/rtree/rtree6.test2
-rw-r--r--ext/rtree/rtree7.test28
-rw-r--r--ext/rtree/rtree9.test1
-rw-r--r--ext/rtree/rtreeB.test37
-rw-r--r--ext/rtree/sqlite3rtree.h6
24 files changed, 3929 insertions, 437 deletions
diff --git a/ext/async/sqlite3async.c b/ext/async/sqlite3async.c
index a351eaa..0814da7 100644
--- a/ext/async/sqlite3async.c
+++ b/ext/async/sqlite3async.c
@@ -944,7 +944,7 @@ static int asyncFileControl(sqlite3_file *id, int op, void *pArg){
return SQLITE_OK;
}
}
- return SQLITE_ERROR;
+ return SQLITE_NOTFOUND;
}
/*
@@ -1044,15 +1044,18 @@ static int asyncOpen(
char *z;
int isAsyncOpen = doAsynchronousOpen(flags);
- /* If zName is NULL, then the upper layer is requesting an anonymous file */
+ /* If zName is NULL, then the upper layer is requesting an anonymous file.
+ ** Otherwise, allocate enough space to make a copy of the file name (along
+ ** with the second nul-terminator byte required by xOpen).
+ */
if( zName ){
- nName = (int)strlen(zName)+1;
+ nName = (int)strlen(zName);
}
nByte = (
sizeof(AsyncFileData) + /* AsyncFileData structure */
2 * pVfs->szOsFile + /* AsyncFileData.pBaseRead and pBaseWrite */
- nName /* AsyncFileData.zName */
+ nName + 2 /* AsyncFileData.zName */
);
z = sqlite3_malloc(nByte);
if( !z ){
diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index 12013f2..f80e303 100644
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -70,7 +70,7 @@
** A doclist is stored like this:
**
** array {
-** varint docid;
+** varint docid; (delta from previous doclist)
** array { (position list for column 0)
** varint position; (2 more than the delta from previous position)
** }
@@ -101,8 +101,8 @@
** at D signals the start of a new column; the 1 at E indicates that the
** new column is column number 1. There are two positions at 12 and 45
** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The
-** 234 at I is the next docid. It has one position 72 (72-2) and then
-** terminates with the 0 at K.
+** 234 at I is the delta to next docid (357). It has one position 70
+** (72-2) and then terminates with the 0 at K.
**
** A "position-list" is the list of positions for multiple columns for
** a single docid. A "column-list" is the set of positions for a single
@@ -286,10 +286,6 @@
** will eventually overtake the earlier data and knock it out. The
** query logic likewise merges doclists so that newer data knocks out
** older data.
-**
-** TODO(shess) Provide a VACUUM type operation to clear out all
-** deletions and duplications. This would basically be a forced merge
-** into a single segment.
*/
#include "fts3Int.h"
@@ -469,6 +465,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
sqlite3_free(p->zReadExprlist);
sqlite3_free(p->zWriteExprlist);
sqlite3_free(p->zContentTbl);
+ sqlite3_free(p->zLanguageid);
/* Invoke the tokenizer destructor to free the tokenizer. */
p->pTokenizer->pModule->xDestroy(p->pTokenizer);
@@ -545,7 +542,9 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){
int rc; /* Return code */
char *zSql; /* SQL statement passed to declare_vtab() */
char *zCols; /* List of user defined columns */
+ const char *zLanguageid;
+ zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid");
sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
/* Create a list of user columns for the virtual table */
@@ -556,7 +555,8 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){
/* Create the whole "CREATE TABLE" statement to pass to SQLite */
zSql = sqlite3_mprintf(
- "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
+ "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)",
+ zCols, p->zName, zLanguageid
);
if( !zCols || !zSql ){
rc = SQLITE_NOMEM;
@@ -571,6 +571,18 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){
}
/*
+** Create the %_stat table if it does not already exist.
+*/
+void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){
+ fts3DbExec(pRc, p->db,
+ "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'"
+ "(id INTEGER PRIMARY KEY, value BLOB);",
+ p->zDb, p->zName
+ );
+ if( (*pRc)==SQLITE_OK ) p->bHasStat = 1;
+}
+
+/*
** Create the backing store tables (%_content, %_segments and %_segdir)
** required by the FTS3 table passed as the only argument. This is done
** as part of the vtab xCreate() method.
@@ -585,6 +597,7 @@ static int fts3CreateTables(Fts3Table *p){
sqlite3 *db = p->db; /* The database connection */
if( p->zContentTbl==0 ){
+ const char *zLanguageid = p->zLanguageid;
char *zContentCols; /* Columns of %_content table */
/* Create a list of user columns for the content table */
@@ -593,6 +606,9 @@ static int fts3CreateTables(Fts3Table *p){
char *z = p->azColumn[i];
zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
}
+ if( zLanguageid && zContentCols ){
+ zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid);
+ }
if( zContentCols==0 ) rc = SQLITE_NOMEM;
/* Create the content table */
@@ -626,11 +642,9 @@ static int fts3CreateTables(Fts3Table *p){
p->zDb, p->zName
);
}
+ assert( p->bHasStat==p->bFts4 );
if( p->bHasStat ){
- fts3DbExec(&rc, db,
- "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);",
- p->zDb, p->zName
- );
+ sqlite3Fts3CreateStatTable(&rc, p);
}
return rc;
}
@@ -712,6 +726,7 @@ static void fts3Appendf(
char *z;
va_start(ap, zFormat);
z = sqlite3_vmprintf(zFormat, ap);
+ va_end(ap);
if( z && *pz ){
char *z2 = sqlite3_mprintf("%s%s", *pz, z);
sqlite3_free(z);
@@ -736,7 +751,7 @@ static void fts3Appendf(
static char *fts3QuoteId(char const *zInput){
int nRet;
char *zRet;
- nRet = 2 + strlen(zInput)*2 + 1;
+ nRet = 2 + (int)strlen(zInput)*2 + 1;
zRet = sqlite3_malloc(nRet);
if( zRet ){
int i;
@@ -791,14 +806,20 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
for(i=0; i<p->nColumn; i++){
fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
}
+ if( p->zLanguageid ){
+ fts3Appendf(pRc, &zRet, ", x.%Q", "langid");
+ }
sqlite3_free(zFree);
}else{
fts3Appendf(pRc, &zRet, "rowid");
for(i=0; i<p->nColumn; i++){
fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]);
}
+ if( p->zLanguageid ){
+ fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid);
+ }
}
- fts3Appendf(pRc, &zRet, "FROM '%q'.'%q%s' AS x",
+ fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x",
p->zDb,
(p->zContentTbl ? p->zContentTbl : p->zName),
(p->zContentTbl ? "" : "_content")
@@ -841,6 +862,9 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){
for(i=0; i<p->nColumn; i++){
fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
}
+ if( p->zLanguageid ){
+ fts3Appendf(pRc, &zRet, ", ?");
+ }
sqlite3_free(zFree);
return zRet;
}
@@ -983,7 +1007,7 @@ static int fts3ContentColumns(
nCol = sqlite3_column_count(pStmt);
for(i=0; i<nCol; i++){
const char *zCol = sqlite3_column_name(pStmt, i);
- nStr += strlen(zCol) + 1;
+ nStr += (int)strlen(zCol) + 1;
}
/* Allocate and populate the array to return. */
@@ -994,7 +1018,7 @@ static int fts3ContentColumns(
char *p = (char *)&azCol[nCol];
for(i=0; i<nCol; i++){
const char *zCol = sqlite3_column_name(pStmt, i);
- int n = strlen(zCol)+1;
+ int n = (int)strlen(zCol)+1;
memcpy(p, zCol, n);
azCol[i] = p;
p += n;
@@ -1056,6 +1080,7 @@ static int fts3InitVtab(
char *zCompress = 0; /* compress=? parameter (or NULL) */
char *zUncompress = 0; /* uncompress=? parameter (or NULL) */
char *zContent = 0; /* content=? parameter (or NULL) */
+ char *zLanguageid = 0; /* languageid=? parameter (or NULL) */
assert( strlen(argv[0])==4 );
assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
@@ -1105,7 +1130,8 @@ static int fts3InitVtab(
{ "compress", 8 }, /* 2 -> COMPRESS */
{ "uncompress", 10 }, /* 3 -> UNCOMPRESS */
{ "order", 5 }, /* 4 -> ORDER */
- { "content", 7 } /* 5 -> CONTENT */
+ { "content", 7 }, /* 5 -> CONTENT */
+ { "languageid", 10 } /* 6 -> LANGUAGEID */
};
int iOpt;
@@ -1159,12 +1185,18 @@ static int fts3InitVtab(
bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
break;
- default: /* CONTENT */
- assert( iOpt==5 );
- sqlite3_free(zUncompress);
+ case 5: /* CONTENT */
+ sqlite3_free(zContent);
zContent = zVal;
zVal = 0;
break;
+
+ case 6: /* LANGUAGEID */
+ assert( iOpt==6 );
+ sqlite3_free(zLanguageid);
+ zLanguageid = zVal;
+ zVal = 0;
+ break;
}
}
sqlite3_free(zVal);
@@ -1194,8 +1226,21 @@ static int fts3InitVtab(
sqlite3_free((void*)aCol);
aCol = 0;
rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString);
+
+ /* If a languageid= option was specified, remove the language id
+ ** column from the aCol[] array. */
+ if( rc==SQLITE_OK && zLanguageid ){
+ int j;
+ for(j=0; j<nCol; j++){
+ if( sqlite3_stricmp(zLanguageid, aCol[j])==0 ){
+ int k;
+ for(k=j; k<nCol; k++) aCol[k] = aCol[k+1];
+ nCol--;
+ break;
+ }
+ }
+ }
}
- assert( rc!=SQLITE_OK || nCol>0 );
}
if( rc!=SQLITE_OK ) goto fts3_init_out;
@@ -1240,9 +1285,13 @@ static int fts3InitVtab(
p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
p->bHasDocsize = (isFts4 && bNoDocsize==0);
p->bHasStat = isFts4;
+ p->bFts4 = isFts4;
p->bDescIdx = bDescIdx;
+ p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */
p->zContentTbl = zContent;
+ p->zLanguageid = zLanguageid;
zContent = 0;
+ zLanguageid = 0;
TESTONLY( p->inTransaction = -1 );
TESTONLY( p->mxSavepoint = -1 );
@@ -1291,6 +1340,16 @@ static int fts3InitVtab(
rc = fts3CreateTables(p);
}
+ /* Check to see if a legacy fts3 table has been "upgraded" by the
+ ** addition of a %_stat table so that it can use incremental merge.
+ */
+ if( !isFts4 && !isCreate ){
+ int rc2 = SQLITE_OK;
+ fts3DbExec(&rc2, db, "SELECT 1 FROM %Q.'%q_stat' WHERE id=2",
+ p->zDb, p->zName);
+ if( rc2==SQLITE_OK ) p->bHasStat = 1;
+ }
+
/* Figure out the page-size for the database. This is required in order to
** estimate the cost of loading large doclists from the database. */
fts3DatabasePageSize(&rc, p);
@@ -1305,6 +1364,7 @@ fts3_init_out:
sqlite3_free(zCompress);
sqlite3_free(zUncompress);
sqlite3_free(zContent);
+ sqlite3_free(zLanguageid);
sqlite3_free((void *)aCol);
if( rc!=SQLITE_OK ){
if( p ){
@@ -1356,6 +1416,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
Fts3Table *p = (Fts3Table *)pVTab;
int i; /* Iterator variable */
int iCons = -1; /* Index of constraint to use */
+ int iLangidCons = -1; /* Index of langid=x constraint, if present */
/* By default use a full table scan. This is an expensive option,
** so search through the constraints to see if a more efficient
@@ -1368,7 +1429,8 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
if( pCons->usable==0 ) continue;
/* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */
- if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ
+ if( iCons<0
+ && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ
&& (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 )
){
pInfo->idxNum = FTS3_DOCID_SEARCH;
@@ -1391,7 +1453,13 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn;
pInfo->estimatedCost = 2.0;
iCons = i;
- break;
+ }
+
+ /* Equality constraint on the langid column */
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ
+ && pCons->iColumn==p->nColumn + 2
+ ){
+ iLangidCons = i;
}
}
@@ -1399,6 +1467,9 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
pInfo->aConstraintUsage[iCons].argvIndex = 1;
pInfo->aConstraintUsage[iCons].omit = 1;
}
+ if( iLangidCons>=0 ){
+ pInfo->aConstraintUsage[iLangidCons].argvIndex = 2;
+ }
/* Regardless of the strategy selected, FTS can deliver rows in rowid (or
** docid) order. Both ascending and descending are possible.
@@ -2282,7 +2353,7 @@ static int fts3DoclistOrMerge(
}
*paOut = aOut;
- *pnOut = (p-aOut);
+ *pnOut = (int)(p-aOut);
assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 );
return SQLITE_OK;
}
@@ -2346,7 +2417,7 @@ static void fts3DoclistPhraseMerge(
}
}
- *pnRight = p - aOut;
+ *pnRight = (int)(p - aOut);
}
/*
@@ -2548,6 +2619,7 @@ static int fts3SegReaderCursorAppend(
*/
static int fts3SegReaderCursor(
Fts3Table *p, /* FTS3 table handle */
+ int iLangid, /* Language id */
int iIndex, /* Index to search (from 0 to p->nIndex-1) */
int iLevel, /* Level of segments to scan */
const char *zTerm, /* Term to query for */
@@ -2576,7 +2648,7 @@ static int fts3SegReaderCursor(
if( iLevel!=FTS3_SEGCURSOR_PENDING ){
if( rc==SQLITE_OK ){
- rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt);
+ rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt);
}
while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
@@ -2599,7 +2671,9 @@ static int fts3SegReaderCursor(
}
rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1,
- iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg
+ (isPrefix==0 && isScan==0),
+ iStartBlock, iLeavesEndBlock,
+ iEndBlock, zRoot, nRoot, &pSeg
);
if( rc!=SQLITE_OK ) goto finished;
rc = fts3SegReaderCursorAppend(pCsr, pSeg);
@@ -2619,6 +2693,7 @@ static int fts3SegReaderCursor(
*/
int sqlite3Fts3SegReaderCursor(
Fts3Table *p, /* FTS3 table handle */
+ int iLangid, /* Language-id to search */
int iIndex, /* Index to search (from 0 to p->nIndex-1) */
int iLevel, /* Level of segments to scan */
const char *zTerm, /* Term to query for */
@@ -2636,14 +2711,9 @@ int sqlite3Fts3SegReaderCursor(
assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
assert( isPrefix==0 || isScan==0 );
- /* "isScan" is only set to true by the ft4aux module, an ordinary
- ** full-text tables. */
- assert( isScan==0 || p->aIndex==0 );
-
memset(pCsr, 0, sizeof(Fts3MultiSegReader));
-
return fts3SegReaderCursor(
- p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
+ p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
);
}
@@ -2655,11 +2725,14 @@ int sqlite3Fts3SegReaderCursor(
*/
static int fts3SegReaderCursorAddZero(
Fts3Table *p, /* FTS virtual table handle */
+ int iLangid,
const char *zTerm, /* Term to scan doclist of */
int nTerm, /* Number of bytes in zTerm */
Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */
){
- return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
+ return fts3SegReaderCursor(p,
+ iLangid, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr
+ );
}
/*
@@ -2695,8 +2768,9 @@ static int fts3TermSegReaderCursor(
for(i=1; bFound==0 && i<p->nIndex; i++){
if( p->aIndex[i].nPrefix==nTerm ){
bFound = 1;
- rc = sqlite3Fts3SegReaderCursor(
- p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr);
+ rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid,
+ i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr
+ );
pSegcsr->bLookup = 1;
}
}
@@ -2704,19 +2778,21 @@ static int fts3TermSegReaderCursor(
for(i=1; bFound==0 && i<p->nIndex; i++){
if( p->aIndex[i].nPrefix==nTerm+1 ){
bFound = 1;
- rc = sqlite3Fts3SegReaderCursor(
- p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr
+ rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid,
+ i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr
);
if( rc==SQLITE_OK ){
- rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr);
+ rc = fts3SegReaderCursorAddZero(
+ p, pCsr->iLangid, zTerm, nTerm, pSegcsr
+ );
}
}
}
}
if( bFound==0 ){
- rc = sqlite3Fts3SegReaderCursor(
- p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
+ rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid,
+ 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
);
pSegcsr->bLookup = !isPrefix;
}
@@ -2871,7 +2947,7 @@ static int fts3FilterMethod(
UNUSED_PARAMETER(nVal);
assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
- assert( nVal==0 || nVal==1 );
+ assert( nVal==0 || nVal==1 || nVal==2 );
assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
assert( p->pSegments==0 );
@@ -2896,8 +2972,11 @@ static int fts3FilterMethod(
return SQLITE_NOMEM;
}
- rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat,
- p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
+ pCsr->iLangid = 0;
+ if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]);
+
+ rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid,
+ p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
);
if( rc!=SQLITE_OK ){
if( rc==SQLITE_ERROR ){
@@ -2968,10 +3047,17 @@ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
/*
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
+**
+** If:
+**
+** (iCol < p->nColumn) -> The value of the iCol'th user column.
+** (iCol == p->nColumn) -> Magic column with the same name as the table.
+** (iCol == p->nColumn+1) -> Docid column
+** (iCol == p->nColumn+2) -> Langid column
*/
static int fts3ColumnMethod(
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
- sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
+ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
int iCol /* Index of column to read value from */
){
int rc = SQLITE_OK; /* Return Code */
@@ -2979,22 +3065,34 @@ static int fts3ColumnMethod(
Fts3Table *p = (Fts3Table *)pCursor->pVtab;
/* The column value supplied by SQLite must be in range. */
- assert( iCol>=0 && iCol<=p->nColumn+1 );
+ assert( iCol>=0 && iCol<=p->nColumn+2 );
if( iCol==p->nColumn+1 ){
/* This call is a request for the "docid" column. Since "docid" is an
** alias for "rowid", use the xRowid() method to obtain the value.
*/
- sqlite3_result_int64(pContext, pCsr->iPrevId);
+ sqlite3_result_int64(pCtx, pCsr->iPrevId);
}else if( iCol==p->nColumn ){
/* The extra column whose name is the same as the table.
- ** Return a blob which is a pointer to the cursor.
- */
- sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
+ ** Return a blob which is a pointer to the cursor. */
+ sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
+ }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
+ sqlite3_result_int64(pCtx, pCsr->iLangid);
}else{
+ /* The requested column is either a user column (one that contains
+ ** indexed data), or the language-id column. */
rc = fts3CursorSeek(0, pCsr);
- if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
- sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
+
+ if( rc==SQLITE_OK ){
+ if( iCol==p->nColumn+2 ){
+ int iLangid = 0;
+ if( p->zLanguageid ){
+ iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1);
+ }
+ sqlite3_result_int(pCtx, iLangid);
+ }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
+ sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
+ }
}
}
@@ -3021,8 +3119,42 @@ static int fts3UpdateMethod(
** hash-table to the database.
*/
static int fts3SyncMethod(sqlite3_vtab *pVtab){
- int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
- sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
+
+ /* Following an incremental-merge operation, assuming that the input
+ ** segments are not completely consumed (the usual case), they are updated
+ ** in place to remove the entries that have already been merged. This
+ ** involves updating the leaf block that contains the smallest unmerged
+ ** entry and each block (if any) between the leaf and the root node. So
+ ** if the height of the input segment b-trees is N, and input segments
+ ** are merged eight at a time, updating the input segments at the end
+ ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually
+ ** small - often between 0 and 2. So the overhead of the incremental
+ ** merge is somewhere between 8 and 24 blocks. To avoid this overhead
+ ** dwarfing the actual productive work accomplished, the incremental merge
+ ** is only attempted if it will write at least 64 leaf blocks. Hence
+ ** nMinMerge.
+ **
+ ** Of course, updating the input segments also involves deleting a bunch
+ ** of blocks from the segments table. But this is not considered overhead
+ ** as it would also be required by a crisis-merge that used the same input
+ ** segments.
+ */
+ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */
+
+ Fts3Table *p = (Fts3Table*)pVtab;
+ int rc = sqlite3Fts3PendingTermsFlush(p);
+
+ if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){
+ int mxLevel = 0; /* Maximum relative level value in db */
+ int A; /* Incr-merge parameter A */
+
+ rc = sqlite3Fts3MaxLevel(p, &mxLevel);
+ assert( rc==SQLITE_OK || mxLevel==0 );
+ A = p->nLeafAdd * mxLevel;
+ A += (A/2);
+ if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8);
+ }
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
@@ -3030,13 +3162,14 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){
** Implementation of xBegin() method. This is a no-op.
*/
static int fts3BeginMethod(sqlite3_vtab *pVtab){
- TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
+ Fts3Table *p = (Fts3Table*)pVtab;
UNUSED_PARAMETER(pVtab);
assert( p->pSegments==0 );
assert( p->nPendingData==0 );
assert( p->inTransaction!=1 );
TESTONLY( p->inTransaction = 1 );
TESTONLY( p->mxSavepoint = -1; );
+ p->nLeafAdd = 0;
return SQLITE_OK;
}
@@ -3331,11 +3464,15 @@ static int fts3RenameMethod(
** Flush the contents of the pending-terms table to disk.
*/
static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
+ int rc = SQLITE_OK;
UNUSED_PARAMETER(iSavepoint);
assert( ((Fts3Table *)pVtab)->inTransaction );
assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
- return fts3SyncMethod(pVtab);
+ if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
+ rc = fts3SyncMethod(pVtab);
+ }
+ return rc;
}
/*
@@ -3695,7 +3832,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
sqlite3_free(aPoslist);
aPoslist = pList;
- nPoslist = aOut - aPoslist;
+ nPoslist = (int)(aOut - aPoslist);
if( nPoslist==0 ){
sqlite3_free(aPoslist);
pPhrase->doclist.pList = 0;
@@ -3739,7 +3876,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
pPhrase->doclist.pList = aOut;
if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
pPhrase->doclist.bFreeList = 1;
- pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList);
+ pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList);
}else{
sqlite3_free(aOut);
pPhrase->doclist.pList = 0;
@@ -3808,7 +3945,7 @@ void sqlite3Fts3DoclistPrev(
int nDoclist, /* Length of aDoclist in bytes */
char **ppIter, /* IN/OUT: Iterator pointer */
sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
- int *pnList, /* IN/OUT: List length pointer */
+ int *pnList, /* OUT: List length pointer */
u8 *pbEof /* OUT: End-of-file flag */
){
char *p = *ppIter;
@@ -3835,7 +3972,7 @@ void sqlite3Fts3DoclistPrev(
iMul = (bDescIdx ? -1 : 1);
}
- *pnList = pEnd - pNext;
+ *pnList = (int)(pEnd - pNext);
*ppIter = pNext;
*piDocid = iDocid;
}else{
@@ -3849,13 +3986,48 @@ void sqlite3Fts3DoclistPrev(
}else{
char *pSave = p;
fts3ReversePoslist(aDoclist, &p);
- *pnList = (pSave - p);
+ *pnList = (int)(pSave - p);
}
*ppIter = p;
}
}
/*
+** Iterate forwards through a doclist.
+*/
+void sqlite3Fts3DoclistNext(
+ int bDescIdx, /* True if the doclist is desc */
+ char *aDoclist, /* Pointer to entire doclist */
+ int nDoclist, /* Length of aDoclist in bytes */
+ char **ppIter, /* IN/OUT: Iterator pointer */
+ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
+ u8 *pbEof /* OUT: End-of-file flag */
+){
+ char *p = *ppIter;
+
+ assert( nDoclist>0 );
+ assert( *pbEof==0 );
+ assert( p || *piDocid==0 );
+ assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) );
+
+ if( p==0 ){
+ p = aDoclist;
+ p += sqlite3Fts3GetVarint(p, piDocid);
+ }else{
+ fts3PoslistCopy(0, &p);
+ if( p>=&aDoclist[nDoclist] ){
+ *pbEof = 1;
+ }else{
+ sqlite3_int64 iVar;
+ p += sqlite3Fts3GetVarint(p, &iVar);
+ *piDocid += ((bDescIdx ? -1 : 1) * iVar);
+ }
+ }
+
+ *ppIter = p;
+}
+
+/*
** Attempt to move the phrase iterator to point to the next matching docid.
** If an error occurs, return an SQLite error code. Otherwise, return
** SQLITE_OK.
@@ -3909,7 +4081,7 @@ static int fts3EvalPhraseNext(
}
pDL->pList = pIter;
fts3PoslistCopy(0, &pIter);
- pDL->nList = (pIter - pDL->pList);
+ pDL->nList = (int)(pIter - pDL->pList);
/* pIter now points just past the 0x00 that terminates the position-
** list for document pDL->iDocid. However, if this position-list was
@@ -4250,7 +4422,7 @@ static int fts3EvalStart(Fts3Cursor *pCsr){
fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc);
/* Determine which, if any, tokens in the expression should be deferred. */
- if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){
+ if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){
Fts3TokenAndCost *aTC;
Fts3Expr **apOr;
aTC = (Fts3TokenAndCost *)sqlite3_malloc(
@@ -4267,8 +4439,8 @@ static int fts3EvalStart(Fts3Cursor *pCsr){
Fts3Expr **ppOr = apOr;
fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc);
- nToken = pTC-aTC;
- nOr = ppOr-apOr;
+ nToken = (int)(pTC-aTC);
+ nOr = (int)(ppOr-apOr);
if( rc==SQLITE_OK ){
rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
@@ -4340,7 +4512,7 @@ static int fts3EvalNearTrim(
&pOut, aTmp, nParam1, nParam2, paPoslist, &p2
);
if( res ){
- nNew = (pOut - pPhrase->doclist.pList) - 1;
+ nNew = (int)(pOut - pPhrase->doclist.pList) - 1;
assert( pPhrase->doclist.pList[nNew]=='\0' );
assert( nNew<=pPhrase->doclist.nList && nNew>0 );
memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew);
@@ -5010,26 +5182,87 @@ int sqlite3Fts3EvalPhraseStats(
** This function works regardless of whether or not the phrase is deferred,
** incremental, or neither.
*/
-char *sqlite3Fts3EvalPhrasePoslist(
+int sqlite3Fts3EvalPhrasePoslist(
Fts3Cursor *pCsr, /* FTS3 cursor object */
Fts3Expr *pExpr, /* Phrase to return doclist for */
- int iCol /* Column to return position list for */
+ int iCol, /* Column to return position list for */
+ char **ppOut /* OUT: Pointer to position list */
){
Fts3Phrase *pPhrase = pExpr->pPhrase;
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
- char *pIter = pPhrase->doclist.pList;
+ char *pIter;
int iThis;
+ sqlite3_int64 iDocid;
+ /* If this phrase is applies specifically to some column other than
+ ** column iCol, return a NULL pointer. */
+ *ppOut = 0;
assert( iCol>=0 && iCol<pTab->nColumn );
- if( !pIter
- || pExpr->bEof
- || pExpr->iDocid!=pCsr->iPrevId
- || (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol)
- ){
- return 0;
+ if( (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) ){
+ return SQLITE_OK;
}
- assert( pPhrase->doclist.nList>0 );
+ iDocid = pExpr->iDocid;
+ pIter = pPhrase->doclist.pList;
+ if( iDocid!=pCsr->iPrevId || pExpr->bEof ){
+ int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */
+ int bOr = 0;
+ u8 bEof = 0;
+ Fts3Expr *p;
+
+ /* Check if this phrase descends from an OR expression node. If not,
+ ** return NULL. Otherwise, the entry that corresponds to docid
+ ** pCsr->iPrevId may lie earlier in the doclist buffer. */
+ for(p=pExpr->pParent; p; p=p->pParent){
+ if( p->eType==FTSQUERY_OR ) bOr = 1;
+ }
+ if( bOr==0 ) return SQLITE_OK;
+
+ /* This is the descendent of an OR node. In this case we cannot use
+ ** an incremental phrase. Load the entire doclist for the phrase
+ ** into memory in this case. */
+ if( pPhrase->bIncr ){
+ int rc = SQLITE_OK;
+ int bEofSave = pExpr->bEof;
+ fts3EvalRestart(pCsr, pExpr, &rc);
+ while( rc==SQLITE_OK && !pExpr->bEof ){
+ fts3EvalNextRow(pCsr, pExpr, &rc);
+ if( bEofSave==0 && pExpr->iDocid==iDocid ) break;
+ }
+ pIter = pPhrase->doclist.pList;
+ assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
+ if( rc!=SQLITE_OK ) return rc;
+ }
+
+ if( pExpr->bEof ){
+ pIter = 0;
+ iDocid = 0;
+ }
+ bEof = (pPhrase->doclist.nAll==0);
+ assert( bDescDoclist==0 || bDescDoclist==1 );
+ assert( pCsr->bDesc==0 || pCsr->bDesc==1 );
+
+ if( pCsr->bDesc==bDescDoclist ){
+ int dummy;
+ while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
+ sqlite3Fts3DoclistPrev(
+ bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
+ &pIter, &iDocid, &dummy, &bEof
+ );
+ }
+ }else{
+ while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
+ sqlite3Fts3DoclistNext(
+ bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
+ &pIter, &iDocid, &bEof
+ );
+ }
+ }
+
+ if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
+ }
+ if( pIter==0 ) return SQLITE_OK;
+
if( *pIter==0x01 ){
pIter++;
pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
@@ -5043,7 +5276,8 @@ char *sqlite3Fts3EvalPhrasePoslist(
pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
}
- return ((iCol==iThis)?pIter:0);
+ *ppOut = ((iCol==iThis)?pIter:0);
+ return SQLITE_OK;
}
/*
@@ -5066,6 +5300,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
}
}
+
/*
** Return SQLITE_CORRUPT_VTAB.
*/
diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h
index 78392ec..a60b4b6 100644
--- a/ext/fts3/fts3Int.h
+++ b/ext/fts3/fts3Int.h
@@ -67,6 +67,9 @@ extern const sqlite3_api_routines *sqlite3_api;
#ifndef MIN
# define MIN(x,y) ((x)<(y)?(x):(y))
#endif
+#ifndef MAX
+# define MAX(x,y) ((x)>(y)?(x):(y))
+#endif
/*
** Maximum length of a varint encoded integer. The varint format is different
@@ -121,7 +124,7 @@ extern const sqlite3_api_routines *sqlite3_api;
# define NEVER(X) (0)
#else
# define ALWAYS(x) (x)
-# define NEVER(X) (x)
+# define NEVER(x) (x)
#endif
/*
@@ -131,6 +134,7 @@ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */
typedef short int i16; /* 2-byte (or larger) signed integer */
typedef unsigned int u32; /* 4-byte unsigned integer */
typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */
+typedef sqlite3_int64 i64; /* 8-byte signed integer */
/*
** Macro used to suppress compiler warnings for unused parameters.
@@ -192,36 +196,44 @@ struct Fts3Table {
char **azColumn; /* column names. malloced */
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
char *zContentTbl; /* content=xxx option, or NULL */
+ char *zLanguageid; /* languageid=xxx option, or NULL */
+ u8 bAutoincrmerge; /* True if automerge=1 */
+ u32 nLeafAdd; /* Number of leaf blocks added this trans */
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
*/
- sqlite3_stmt *aStmt[27];
+ sqlite3_stmt *aStmt[37];
char *zReadExprlist;
char *zWriteExprlist;
int nNodeSize; /* Soft limit for node size */
+ u8 bFts4; /* True for FTS4, false for FTS3 */
u8 bHasStat; /* True if %_stat table exists */
u8 bHasDocsize; /* True if %_docsize table exists */
u8 bDescIdx; /* True if doclists are in reverse order */
+ u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */
int nPgsz; /* Page size for host database */
char *zSegmentsTbl; /* Name of %_segments table */
sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
- /* TODO: Fix the first paragraph of this comment.
- **
- ** The following hash table is used to buffer pending index updates during
- ** transactions. Variable nPendingData estimates the memory size of the
- ** pending data, including hash table overhead, but not malloc overhead.
- ** When nPendingData exceeds nMaxPendingData, the buffer is flushed
- ** automatically. Variable iPrevDocid is the docid of the most recently
- ** inserted record.
+ /*
+ ** The following array of hash tables is used to buffer pending index
+ ** updates during transactions. All pending updates buffered at any one
+ ** time must share a common language-id (see the FTS4 langid= feature).
+ ** The current language id is stored in variable iPrevLangid.
**
** A single FTS4 table may have multiple full-text indexes. For each index
** there is an entry in the aIndex[] array. Index 0 is an index of all the
** terms that appear in the document set. Each subsequent index in aIndex[]
** is an index of prefixes of a specific length.
+ **
+ ** Variable nPendingData contains an estimate the memory consumed by the
+ ** pending data structures, including hash table overhead, but not including
+ ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash
+ ** tables are flushed to disk. Variable iPrevDocid is the docid of the most
+ ** recently inserted record.
*/
int nIndex; /* Size of aIndex[] */
struct Fts3Index {
@@ -231,12 +243,13 @@ struct Fts3Table {
int nMaxPendingData; /* Max pending data before flush to disk */
int nPendingData; /* Current bytes of pending data */
sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
+ int iPrevLangid; /* Langid of recently inserted document */
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
/* State variables used for validating that the transaction control
** methods of the virtual table are called at appropriate times. These
- ** values do not contribution to the FTS computation; they are used for
- ** verifying the SQLite core.
+ ** values do not contribute to FTS functionality; they are used for
+ ** verifying the operation of the SQLite core.
*/
int inTransaction; /* True after xBegin but before xCommit/xRollback */
int mxSavepoint; /* Largest valid xSavepoint integer */
@@ -255,6 +268,7 @@ struct Fts3Cursor {
u8 isRequireSeek; /* True if must seek pStmt to %_content row */
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
Fts3Expr *pExpr; /* Parsed MATCH query string */
+ int iLangid; /* Language being queried for */
int nPhrase; /* Number of matchable phrases in query */
Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
@@ -401,12 +415,12 @@ int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
int sqlite3Fts3PendingTermsFlush(Fts3Table *);
void sqlite3Fts3PendingTermsClear(Fts3Table *);
int sqlite3Fts3Optimize(Fts3Table *);
-int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
+int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64,
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
int sqlite3Fts3SegReaderPending(
Fts3Table*,int,const char*,int,int,Fts3SegReader**);
void sqlite3Fts3SegReaderFree(Fts3SegReader *);
-int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
+int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **);
int sqlite3Fts3ReadLock(Fts3Table *);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
@@ -418,6 +432,7 @@ int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
void sqlite3Fts3SegmentsClose(Fts3Table *);
+int sqlite3Fts3MaxLevel(Fts3Table *, int *);
/* Special values interpreted by sqlite3SegReaderCursor() */
#define FTS3_SEGCURSOR_PENDING -1
@@ -427,8 +442,8 @@ int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
-int sqlite3Fts3SegReaderCursor(
- Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
+int sqlite3Fts3SegReaderCursor(Fts3Table *,
+ int, int, int, const char *, int, int, int, Fts3MultiSegReader *);
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
@@ -469,6 +484,8 @@ struct Fts3MultiSegReader {
int nDoclist; /* Size of aDoclist[] in bytes */
};
+int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
+
/* fts3.c */
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
@@ -478,6 +495,7 @@ void sqlite3Fts3Dequote(char *);
void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
+void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
@@ -495,7 +513,7 @@ void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
/* fts3_expr.c */
-int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
+int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
char **, int, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
@@ -504,6 +522,10 @@ int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
int sqlite3Fts3InitTerm(sqlite3 *db);
#endif
+int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int,
+ sqlite3_tokenizer_cursor **
+);
+
/* fts3_aux.c */
int sqlite3Fts3InitAux(sqlite3 *db);
@@ -513,7 +535,7 @@ int sqlite3Fts3MsrIncrStart(
Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
int sqlite3Fts3MsrIncrNext(
Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
-char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol);
+int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **);
int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c
index ada85d7..a2bff2e 100644
--- a/ext/fts3/fts3_aux.c
+++ b/ext/fts3/fts3_aux.c
@@ -79,9 +79,9 @@ static int fts3auxConnectMethod(
}
zDb = argv[1];
- nDb = strlen(zDb);
+ nDb = (int)strlen(zDb);
zFts3 = argv[3];
- nFts3 = strlen(zFts3);
+ nFts3 = (int)strlen(zFts3);
rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
if( rc!=SQLITE_OK ) return rc;
@@ -376,7 +376,7 @@ static int fts3auxFilterMethod(
if( pCsr->zStop==0 ) return SQLITE_NOMEM;
}
- rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL,
+ rc = sqlite3Fts3SegReaderCursor(pFts3, 0, 0, FTS3_SEGCURSOR_ALL,
pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
);
if( rc==SQLITE_OK ){
diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c
index 1c3a790..a6e3492 100644
--- a/ext/fts3/fts3_expr.c
+++ b/ext/fts3/fts3_expr.c
@@ -92,6 +92,7 @@ int sqlite3_fts3_enable_parentheses = 0;
typedef struct ParseContext ParseContext;
struct ParseContext {
sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
+ int iLangid; /* Language id used with tokenizer */
const char **azCol; /* Array of column names for fts3 table */
int bFts4; /* True to allow FTS4-only syntax */
int nCol; /* Number of entries in azCol[] */
@@ -127,6 +128,33 @@ static void *fts3MallocZero(int nByte){
return pRet;
}
+int sqlite3Fts3OpenTokenizer(
+ sqlite3_tokenizer *pTokenizer,
+ int iLangid,
+ const char *z,
+ int n,
+ sqlite3_tokenizer_cursor **ppCsr
+){
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ sqlite3_tokenizer_cursor *pCsr = 0;
+ int rc;
+
+ rc = pModule->xOpen(pTokenizer, z, n, &pCsr);
+ assert( rc==SQLITE_OK || pCsr==0 );
+ if( rc==SQLITE_OK ){
+ pCsr->pTokenizer = pTokenizer;
+ if( pModule->iVersion>=1 ){
+ rc = pModule->xLanguageid(pCsr, iLangid);
+ if( rc!=SQLITE_OK ){
+ pModule->xClose(pCsr);
+ pCsr = 0;
+ }
+ }
+ }
+ *ppCsr = pCsr;
+ return rc;
+}
+
/*
** Extract the next token from buffer z (length n) using the tokenizer
@@ -154,15 +182,13 @@ static int getNextToken(
Fts3Expr *pRet = 0;
int nConsumed = 0;
- rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
+ rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor);
if( rc==SQLITE_OK ){
const char *zToken;
int nToken, iStart, iEnd, iPosition;
int nByte; /* total space to allocate */
- pCursor->pTokenizer = pTokenizer;
rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
-
if( rc==SQLITE_OK ){
nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
pRet = (Fts3Expr *)fts3MallocZero(nByte);
@@ -268,10 +294,10 @@ static int getNextString(
** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
** structures.
*/
- rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
+ rc = sqlite3Fts3OpenTokenizer(
+ pTokenizer, pParse->iLangid, zInput, nInput, &pCursor);
if( rc==SQLITE_OK ){
int ii;
- pCursor->pTokenizer = pTokenizer;
for(ii=0; rc==SQLITE_OK; ii++){
const char *zByte;
int nByte, iBegin, iEnd, iPos;
@@ -745,6 +771,7 @@ exprparse_out:
*/
int sqlite3Fts3ExprParse(
sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
+ int iLangid, /* Language id for tokenizer */
char **azCol, /* Array of column names for fts3 table */
int bFts4, /* True to allow FTS4-only syntax */
int nCol, /* Number of entries in azCol[] */
@@ -755,11 +782,13 @@ int sqlite3Fts3ExprParse(
int nParsed;
int rc;
ParseContext sParse;
+
+ memset(&sParse, 0, sizeof(ParseContext));
sParse.pTokenizer = pTokenizer;
+ sParse.iLangid = iLangid;
sParse.azCol = (const char **)azCol;
sParse.nCol = nCol;
sParse.iDefaultCol = iDefaultCol;
- sParse.nNest = 0;
sParse.bFts4 = bFts4;
if( z==0 ){
*ppExpr = 0;
@@ -950,7 +979,7 @@ static void fts3ExprTest(
}
rc = sqlite3Fts3ExprParse(
- pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
+ pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
);
if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3_result_error(context, "Error parsing expression", -1);
diff --git a/ext/fts3/fts3_icu.c b/ext/fts3/fts3_icu.c
index a10a55d..5e9c900 100644
--- a/ext/fts3/fts3_icu.c
+++ b/ext/fts3/fts3_icu.c
@@ -110,7 +110,10 @@ static int icuOpen(
*ppCursor = 0;
- if( nInput<0 ){
+ if( zInput==0 ){
+ nInput = 0;
+ zInput = "";
+ }else if( nInput<0 ){
nInput = strlen(zInput);
}
nChar = nInput+1;
diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c
index 148c570..579745b 100644
--- a/ext/fts3/fts3_porter.c
+++ b/ext/fts3/fts3_porter.c
@@ -40,7 +40,7 @@ typedef struct porter_tokenizer {
} porter_tokenizer;
/*
-** Class derived from sqlit3_tokenizer_cursor
+** Class derived from sqlite3_tokenizer_cursor
*/
typedef struct porter_tokenizer_cursor {
sqlite3_tokenizer_cursor base;
@@ -630,6 +630,7 @@ static const sqlite3_tokenizer_module porterTokenizerModule = {
porterOpen,
porterClose,
porterNext,
+ 0
};
/*
diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c
index 23ef25c..6fce3d0 100644
--- a/ext/fts3/fts3_snippet.c
+++ b/ext/fts3/fts3_snippet.c
@@ -360,10 +360,11 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
SnippetIter *p = (SnippetIter *)ctx;
SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
char *pCsr;
+ int rc;
pPhrase->nToken = pExpr->pPhrase->nToken;
-
- pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
+ rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr);
+ assert( rc==SQLITE_OK || pCsr==0 );
if( pCsr ){
int iFirst = 0;
pPhrase->pList = pCsr;
@@ -374,10 +375,12 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
pPhrase->iHead = iFirst;
pPhrase->iTail = iFirst;
}else{
- assert( pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 );
+ assert( rc!=SQLITE_OK || (
+ pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0
+ ));
}
- return SQLITE_OK;
+ return rc;
}
/*
@@ -532,6 +535,7 @@ static int fts3StringAppend(
*/
static int fts3SnippetShift(
Fts3Table *pTab, /* FTS3 table snippet comes from */
+ int iLangid, /* Language id to use in tokenizing */
int nSnippet, /* Number of tokens desired for snippet */
const char *zDoc, /* Document text to extract snippet from */
int nDoc, /* Size of buffer zDoc in bytes */
@@ -567,11 +571,10 @@ static int fts3SnippetShift(
/* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired)
** or more tokens in zDoc/nDoc.
*/
- rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+ rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC);
if( rc!=SQLITE_OK ){
return rc;
}
- pC->pTokenizer = pTab->pTokenizer;
while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){
const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3;
rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent);
@@ -631,11 +634,10 @@ static int fts3SnippetText(
/* Open a token cursor on the document. */
pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
- rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+ rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC);
if( rc!=SQLITE_OK ){
return rc;
}
- pC->pTokenizer = pTab->pTokenizer;
while( rc==SQLITE_OK ){
int iBegin; /* Offset in zDoc of start of token */
@@ -657,7 +659,9 @@ static int fts3SnippetText(
if( !isShiftDone ){
int n = nDoc - iBegin;
- rc = fts3SnippetShift(pTab, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask);
+ rc = fts3SnippetShift(
+ pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask
+ );
isShiftDone = 1;
/* Now that the shift has been done, check if the initial "..." are
@@ -769,13 +773,14 @@ static int fts3ExprLocalHitsCb(
int iPhrase, /* Phrase number */
void *pCtx /* Pointer to MatchInfo structure */
){
+ int rc = SQLITE_OK;
MatchInfo *p = (MatchInfo *)pCtx;
int iStart = iPhrase * p->nCol * 3;
int i;
- for(i=0; i<p->nCol; i++){
+ for(i=0; i<p->nCol && rc==SQLITE_OK; i++){
char *pCsr;
- pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i);
+ rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr);
if( pCsr ){
p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
}else{
@@ -783,7 +788,7 @@ static int fts3ExprLocalHitsCb(
}
}
- return SQLITE_OK;
+ return rc;
}
static int fts3MatchinfoCheck(
@@ -793,8 +798,8 @@ static int fts3MatchinfoCheck(
){
if( (cArg==FTS3_MATCHINFO_NPHRASE)
|| (cArg==FTS3_MATCHINFO_NCOL)
- || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat)
- || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat)
+ || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4)
+ || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4)
|| (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
|| (cArg==FTS3_MATCHINFO_LCS)
|| (cArg==FTS3_MATCHINFO_HITS)
@@ -944,8 +949,10 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
int nLive = 0; /* Number of iterators in aIter not at EOF */
for(i=0; i<pInfo->nPhrase; i++){
+ int rc;
LcsIterator *pIt = &aIter[i];
- pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol);
+ rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead);
+ if( rc!=SQLITE_OK ) return rc;
if( pIt->pRead ){
pIt->iPos = pIt->iPosOffset;
fts3LcsIteratorAdvance(&aIter[i]);
@@ -1297,9 +1304,10 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
int iTerm; /* For looping through nTerm phrase terms */
char *pList; /* Pointer to position list for phrase */
int iPos = 0; /* First position in position-list */
+ int rc;
UNUSED_PARAMETER(iPhrase);
- pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
+ rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList);
nTerm = pExpr->pPhrase->nToken;
if( pList ){
fts3GetDeltaPosition(&pList, &iPos);
@@ -1313,7 +1321,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
pT->iPos = iPos;
}
- return SQLITE_OK;
+ return rc;
}
/*
@@ -1390,9 +1398,10 @@ void sqlite3Fts3Offsets(
}
/* Initialize a tokenizer iterator to iterate through column iCol. */
- rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+ rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid,
+ zDoc, nDoc, &pC
+ );
if( rc!=SQLITE_OK ) goto offsets_out;
- pC->pTokenizer = pTab->pTokenizer;
rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
while( rc==SQLITE_OK ){
diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c
index d3eb690..c49d5cb 100644
--- a/ext/fts3/fts3_term.c
+++ b/ext/fts3/fts3_term.c
@@ -73,6 +73,7 @@ static int fts3termConnectMethod(
Fts3termTable *p; /* Virtual table object to return */
int iIndex = 0;
+ UNUSED_PARAMETER(pCtx);
if( argc==5 ){
iIndex = atoi(argv[4]);
argc--;
@@ -87,9 +88,9 @@ static int fts3termConnectMethod(
}
zDb = argv[1];
- nDb = strlen(zDb);
+ nDb = (int)strlen(zDb);
zFts3 = argv[3];
- nFts3 = strlen(zFts3);
+ nFts3 = (int)strlen(zFts3);
rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
if( rc!=SQLITE_OK ) return rc;
@@ -231,12 +232,12 @@ static int fts3termNextMethod(sqlite3_vtab_cursor *pCursor){
if( v==1 ){
pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
- pCsr->iCol += v;
+ pCsr->iCol += (int)v;
pCsr->iPos = 0;
pCsr->pNext += sqlite3Fts3GetVarint(pCsr->pNext, &v);
}
- pCsr->iPos += (v - 2);
+ pCsr->iPos += (int)(v - 2);
return SQLITE_OK;
}
@@ -271,7 +272,7 @@ static int fts3termFilterMethod(
pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
- rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL,
+ rc = sqlite3Fts3SegReaderCursor(pFts3, 0, p->iIndex, FTS3_SEGCURSOR_ALL,
pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr
);
if( rc==SQLITE_OK ){
@@ -357,7 +358,10 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
0, /* xCommit */
0, /* xRollback */
0, /* xFindFunction */
- 0 /* xRename */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0 /* xRollbackTo */
};
int rc; /* Return code */
diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c
index 72735f3..4da0b8f 100644
--- a/ext/fts3/fts3_test.c
+++ b/ext/fts3/fts3_test.c
@@ -13,13 +13,17 @@
** This file is not part of the production FTS code. It is only used for
** testing. It contains a Tcl command that can be used to test if a document
** matches an FTS NEAR expression.
+**
+** As of March 2012, it also contains a version 1 tokenizer used for testing
+** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
*/
#include <tcl.h>
#include <string.h>
#include <assert.h>
-#ifdef SQLITE_TEST
+#if defined(SQLITE_TEST)
+#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
/* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */
#include "fts3Int.h"
@@ -158,6 +162,8 @@ static int fts3_near_match_cmd(
Tcl_Obj **apExprToken;
int nExprToken;
+ UNUSED_PARAMETER(clientData);
+
/* Must have 3 or more arguments. */
if( objc<3 || (objc%2)==0 ){
Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?...");
@@ -311,14 +317,219 @@ static int fts3_configure_incr_load_cmd(
Tcl_SetObjResult(interp, pRet);
Tcl_DecrRefCount(pRet);
#endif
+ UNUSED_PARAMETER(clientData);
+ return TCL_OK;
+}
+
+#ifdef SQLITE_ENABLE_FTS3
+/**************************************************************************
+** Beginning of test tokenizer code.
+**
+** For language 0, this tokenizer is similar to the default 'simple'
+** tokenizer. For other languages L, the following:
+**
+** * Odd numbered languages are case-sensitive. Even numbered
+** languages are not.
+**
+** * Language ids 100 or greater are considered an error.
+**
+** The implementation assumes that the input contains only ASCII characters
+** (i.e. those that may be encoded in UTF-8 using a single byte).
+*/
+typedef struct test_tokenizer {
+ sqlite3_tokenizer base;
+} test_tokenizer;
+
+typedef struct test_tokenizer_cursor {
+ sqlite3_tokenizer_cursor base;
+ const char *aInput; /* Input being tokenized */
+ int nInput; /* Size of the input in bytes */
+ int iInput; /* Current offset in aInput */
+ int iToken; /* Index of next token to be returned */
+ char *aBuffer; /* Buffer containing current token */
+ int nBuffer; /* Number of bytes allocated at pToken */
+ int iLangid; /* Configured language id */
+} test_tokenizer_cursor;
+
+static int testTokenizerCreate(
+ int argc, const char * const *argv,
+ sqlite3_tokenizer **ppTokenizer
+){
+ test_tokenizer *pNew;
+ UNUSED_PARAMETER(argc);
+ UNUSED_PARAMETER(argv);
+
+ pNew = sqlite3_malloc(sizeof(test_tokenizer));
+ if( !pNew ) return SQLITE_NOMEM;
+ memset(pNew, 0, sizeof(test_tokenizer));
+
+ *ppTokenizer = (sqlite3_tokenizer *)pNew;
+ return SQLITE_OK;
+}
+
+static int testTokenizerDestroy(sqlite3_tokenizer *pTokenizer){
+ test_tokenizer *p = (test_tokenizer *)pTokenizer;
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+static int testTokenizerOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *pInput, int nBytes, /* String to be tokenized */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+){
+ int rc = SQLITE_OK; /* Return code */
+ test_tokenizer_cursor *pCsr; /* New cursor object */
+
+ UNUSED_PARAMETER(pTokenizer);
+
+ pCsr = (test_tokenizer_cursor *)sqlite3_malloc(sizeof(test_tokenizer_cursor));
+ if( pCsr==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(pCsr, 0, sizeof(test_tokenizer_cursor));
+ pCsr->aInput = pInput;
+ if( nBytes<0 ){
+ pCsr->nInput = (int)strlen(pInput);
+ }else{
+ pCsr->nInput = nBytes;
+ }
+ }
+
+ *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
+ return rc;
+}
+
+static int testTokenizerClose(sqlite3_tokenizer_cursor *pCursor){
+ test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor;
+ sqlite3_free(pCsr->aBuffer);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+static int testIsTokenChar(char c){
+ return (c>='a' && c<='z') || (c>='A' && c<='Z');
+}
+static int testTolower(char c){
+ char ret = c;
+ if( ret>='A' && ret<='Z') ret = ret - ('A'-'a');
+ return ret;
+}
+
+static int testTokenizerNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by testTokenizerOpen */
+ const char **ppToken, /* OUT: *ppToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor;
+ int rc = SQLITE_OK;
+ const char *p;
+ const char *pEnd;
+
+ p = &pCsr->aInput[pCsr->iInput];
+ pEnd = &pCsr->aInput[pCsr->nInput];
+
+ /* Skip past any white-space */
+ assert( p<=pEnd );
+ while( p<pEnd && testIsTokenChar(*p)==0 ) p++;
+
+ if( p==pEnd ){
+ rc = SQLITE_DONE;
+ }else{
+ /* Advance to the end of the token */
+ const char *pToken = p;
+ int nToken;
+ while( p<pEnd && testIsTokenChar(*p) ) p++;
+ nToken = (int)(p-pToken);
+
+ /* Copy the token into the buffer */
+ if( nToken>pCsr->nBuffer ){
+ sqlite3_free(pCsr->aBuffer);
+ pCsr->aBuffer = sqlite3_malloc(nToken);
+ }
+ if( pCsr->aBuffer==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ int i;
+
+ if( pCsr->iLangid & 0x00000001 ){
+ for(i=0; i<nToken; i++) pCsr->aBuffer[i] = pToken[i];
+ }else{
+ for(i=0; i<nToken; i++) pCsr->aBuffer[i] = testTolower(pToken[i]);
+ }
+ pCsr->iToken++;
+ pCsr->iInput = (int)(p - pCsr->aInput);
+
+ *ppToken = pCsr->aBuffer;
+ *pnBytes = nToken;
+ *piStartOffset = (int)(pToken - pCsr->aInput);
+ *piEndOffset = (int)(p - pCsr->aInput);
+ *piPosition = pCsr->iToken;
+ }
+ }
+
+ return rc;
+}
+
+static int testTokenizerLanguage(
+ sqlite3_tokenizer_cursor *pCursor,
+ int iLangid
+){
+ int rc = SQLITE_OK;
+ test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor;
+ pCsr->iLangid = iLangid;
+ if( pCsr->iLangid>=100 ){
+ rc = SQLITE_ERROR;
+ }
+ return rc;
+}
+#endif
+
+static int fts3_test_tokenizer_cmd(
+ ClientData clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+#ifdef SQLITE_ENABLE_FTS3
+ static const sqlite3_tokenizer_module testTokenizerModule = {
+ 1,
+ testTokenizerCreate,
+ testTokenizerDestroy,
+ testTokenizerOpen,
+ testTokenizerClose,
+ testTokenizerNext,
+ testTokenizerLanguage
+ };
+ const sqlite3_tokenizer_module *pPtr = &testTokenizerModule;
+ if( objc!=1 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "");
+ return TCL_ERROR;
+ }
+ Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(
+ (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *)
+ ));
+#endif
+ UNUSED_PARAMETER(clientData);
return TCL_OK;
}
+/*
+** End of tokenizer code.
+**************************************************************************/
+
int Sqlitetestfts3_Init(Tcl_Interp *interp){
Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
Tcl_CreateObjCommand(interp,
"fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
);
+ Tcl_CreateObjCommand(
+ interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0
+ );
return TCL_OK;
}
+#endif /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */
#endif /* ifdef SQLITE_TEST */
diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c
index 6494bb9..f6b044f 100644
--- a/ext/fts3/fts3_tokenizer.c
+++ b/ext/fts3/fts3_tokenizer.c
@@ -288,11 +288,10 @@ static void testFunc(
goto finish;
}
pTokenizer->pModule = p;
- if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
+ if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){
zErr = "error in xOpen()";
goto finish;
}
- pCsr->pTokenizer = pTokenizer;
while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
diff --git a/ext/fts3/fts3_tokenizer.h b/ext/fts3/fts3_tokenizer.h
index 6156445..c91c7ed 100644
--- a/ext/fts3/fts3_tokenizer.h
+++ b/ext/fts3/fts3_tokenizer.h
@@ -52,7 +52,7 @@ typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
struct sqlite3_tokenizer_module {
/*
- ** Structure version. Should always be set to 0.
+ ** Structure version. Should always be set to 0 or 1.
*/
int iVersion;
@@ -133,6 +133,15 @@ struct sqlite3_tokenizer_module {
int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
int *piPosition /* OUT: Number of tokens returned before this one */
);
+
+ /***********************************************************************
+ ** Methods below this point are only available if iVersion>=1.
+ */
+
+ /*
+ ** Configure the language id of a tokenizer cursor.
+ */
+ int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid);
};
struct sqlite3_tokenizer {
diff --git a/ext/fts3/fts3_tokenizer1.c b/ext/fts3/fts3_tokenizer1.c
index d11a499..deea06d 100644
--- a/ext/fts3/fts3_tokenizer1.c
+++ b/ext/fts3/fts3_tokenizer1.c
@@ -218,6 +218,7 @@ static const sqlite3_tokenizer_module simpleTokenizerModule = {
simpleOpen,
simpleClose,
simpleNext,
+ 0,
};
/*
diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 2904a9a..fa5fb02 100644
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -24,6 +24,9 @@
#include <assert.h>
#include <stdlib.h>
+
+#define FTS_MAX_APPENDABLE_HEIGHT 16
+
/*
** When full-text index nodes are loaded from disk, the buffer that they
** are loaded into has the following number of bytes of padding at the end
@@ -63,6 +66,29 @@ int test_fts3_node_chunk_threshold = (4*1024)*4;
# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
#endif
+/*
+** The two values that may be meaningfully bound to the :1 parameter in
+** statements SQL_REPLACE_STAT and SQL_SELECT_STAT.
+*/
+#define FTS_STAT_DOCTOTAL 0
+#define FTS_STAT_INCRMERGEHINT 1
+#define FTS_STAT_AUTOINCRMERGE 2
+
+/*
+** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic
+** and incremental merge operation that takes place. This is used for
+** debugging FTS only, it should not usually be turned on in production
+** systems.
+*/
+#ifdef FTS3_LOG_MERGES
+static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){
+ sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel);
+}
+#else
+#define fts3LogMerge(x, y)
+#endif
+
+
typedef struct PendingList PendingList;
typedef struct SegmentNode SegmentNode;
typedef struct SegmentWriter SegmentWriter;
@@ -110,6 +136,7 @@ struct Fts3DeferredToken {
*/
struct Fts3SegReader {
int iIdx; /* Index within level, or 0x7FFFFFFF for PT */
+ int bLookup; /* True for a lookup only */
sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */
sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */
@@ -223,13 +250,22 @@ struct SegmentNode {
#define SQL_DELETE_DOCSIZE 19
#define SQL_REPLACE_DOCSIZE 20
#define SQL_SELECT_DOCSIZE 21
-#define SQL_SELECT_DOCTOTAL 22
-#define SQL_REPLACE_DOCTOTAL 23
+#define SQL_SELECT_STAT 22
+#define SQL_REPLACE_STAT 23
#define SQL_SELECT_ALL_PREFIX_LEVEL 24
#define SQL_DELETE_ALL_TERMS_SEGDIR 25
-
#define SQL_DELETE_SEGDIR_RANGE 26
+#define SQL_SELECT_ALL_LANGID 27
+#define SQL_FIND_MERGE_LEVEL 28
+#define SQL_MAX_LEAF_NODE_ESTIMATE 29
+#define SQL_DELETE_SEGDIR_ENTRY 30
+#define SQL_SHIFT_SEGDIR_ENTRY 31
+#define SQL_SELECT_SEGDIR 32
+#define SQL_CHOMP_SEGDIR 33
+#define SQL_SEGMENT_IS_APPENDABLE 34
+#define SQL_SELECT_INDEXES 35
+#define SQL_SELECT_MXLEVEL 36
/*
** This function is used to obtain an SQLite prepared statement handle
@@ -258,9 +294,9 @@ static int fts3SqlStmt(
/* 6 */ "DELETE FROM %Q.'%q_stat'",
/* 7 */ "SELECT %s WHERE rowid=?",
/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
-/* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
+/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
-/* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
+/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
/* Return segments in order from oldest to newest.*/
/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
@@ -278,13 +314,61 @@ static int fts3SqlStmt(
/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
-/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
-/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
+/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?",
+/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)",
/* 24 */ "",
/* 25 */ "",
/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
-
+/* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'",
+
+/* This statement is used to determine which level to read the input from
+** when performing an incremental merge. It returns the absolute level number
+** of the oldest level in the db that contains at least ? segments. Or,
+** if no level in the FTS index contains more than ? segments, the statement
+** returns zero rows. */
+/* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?"
+ " ORDER BY (level %% 1024) ASC LIMIT 1",
+
+/* Estimate the upper limit on the number of leaf nodes in a new segment
+** created by merging the oldest :2 segments from absolute level :1. See
+** function sqlite3Fts3Incrmerge() for details. */
+/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) "
+ " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?",
+
+/* SQL_DELETE_SEGDIR_ENTRY
+** Delete the %_segdir entry on absolute level :1 with index :2. */
+/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?",
+
+/* SQL_SHIFT_SEGDIR_ENTRY
+** Modify the idx value for the segment with idx=:3 on absolute level :2
+** to :1. */
+/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?",
+
+/* SQL_SELECT_SEGDIR
+** Read a single entry from the %_segdir table. The entry from absolute
+** level :1 with index value :2. */
+/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?",
+
+/* SQL_CHOMP_SEGDIR
+** Update the start_block (:1) and root (:2) fields of the %_segdir
+** entry located on absolute level :3 with index :4. */
+/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?"
+ "WHERE level = ? AND idx = ?",
+
+/* SQL_SEGMENT_IS_APPENDABLE
+** Return a single row if the segment with end_block=? is appendable. Or
+** no rows otherwise. */
+/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL",
+
+/* SQL_SELECT_INDEXES
+** Return the list of valid segment indexes for absolute level ? */
+/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC",
+
+/* SQL_SELECT_MXLEVEL
+** Return the largest relative level in the FTS index or indexes. */
+/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'"
};
int rc = SQLITE_OK;
sqlite3_stmt *pStmt;
@@ -322,22 +406,18 @@ static int fts3SqlStmt(
return rc;
}
+
static int fts3SelectDocsize(
Fts3Table *pTab, /* FTS3 table handle */
- int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */
sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */
sqlite3_stmt **ppStmt /* OUT: Statement handle */
){
sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */
int rc; /* Return code */
- assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL );
-
- rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0);
+ rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0);
if( rc==SQLITE_OK ){
- if( eStmt==SQL_SELECT_DOCSIZE ){
- sqlite3_bind_int64(pStmt, 1, iDocid);
- }
+ sqlite3_bind_int64(pStmt, 1, iDocid);
rc = sqlite3_step(pStmt);
if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){
rc = sqlite3_reset(pStmt);
@@ -356,7 +436,21 @@ int sqlite3Fts3SelectDoctotal(
Fts3Table *pTab, /* Fts3 table handle */
sqlite3_stmt **ppStmt /* OUT: Statement handle */
){
- return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt);
+ sqlite3_stmt *pStmt = 0;
+ int rc;
+ rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
+ if( sqlite3_step(pStmt)!=SQLITE_ROW
+ || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB
+ ){
+ rc = sqlite3_reset(pStmt);
+ if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB;
+ pStmt = 0;
+ }
+ }
+ *ppStmt = pStmt;
+ return rc;
}
int sqlite3Fts3SelectDocsize(
@@ -364,7 +458,7 @@ int sqlite3Fts3SelectDocsize(
sqlite3_int64 iDocid, /* Docid to read size data for */
sqlite3_stmt **ppStmt /* OUT: Statement handle */
){
- return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt);
+ return fts3SelectDocsize(pTab, iDocid, ppStmt);
}
/*
@@ -431,6 +525,44 @@ int sqlite3Fts3ReadLock(Fts3Table *p){
}
/*
+** FTS maintains a separate indexes for each language-id (a 32-bit integer).
+** Within each language id, a separate index is maintained to store the
+** document terms, and each configured prefix size (configured the FTS
+** "prefix=" option). And each index consists of multiple levels ("relative
+** levels").
+**
+** All three of these values (the language id, the specific index and the
+** level within the index) are encoded in 64-bit integer values stored
+** in the %_segdir table on disk. This function is used to convert three
+** separate component values into the single 64-bit integer value that
+** can be used to query the %_segdir table.
+**
+** Specifically, each language-id/index combination is allocated 1024
+** 64-bit integer level values ("absolute levels"). The main terms index
+** for language-id 0 is allocate values 0-1023. The first prefix index
+** (if any) for language-id 0 is allocated values 1024-2047. And so on.
+** Language 1 indexes are allocated immediately following language 0.
+**
+** So, for a system with nPrefix prefix indexes configured, the block of
+** absolute levels that corresponds to language-id iLangid and index
+** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024).
+*/
+static sqlite3_int64 getAbsoluteLevel(
+ Fts3Table *p, /* FTS3 table handle */
+ int iLangid, /* Language id */
+ int iIndex, /* Index in p->aIndex[] */
+ int iLevel /* Level of segments */
+){
+ sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */
+ assert( iLangid>=0 );
+ assert( p->nIndex>0 );
+ assert( iIndex>=0 && iIndex<p->nIndex );
+
+ iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL;
+ return iBase + iLevel;
+}
+
+/*
** Set *ppStmt to a statement handle that may be used to iterate through
** all rows in the %_segdir table, from oldest to newest. If successful,
** return SQLITE_OK. If an error occurs while preparing the statement,
@@ -449,8 +581,9 @@ int sqlite3Fts3ReadLock(Fts3Table *p){
*/
int sqlite3Fts3AllSegdirs(
Fts3Table *p, /* FTS3 table */
+ int iLangid, /* Language being queried */
int iIndex, /* Index for p->aIndex[] */
- int iLevel, /* Level to select */
+ int iLevel, /* Level to select (relative level) */
sqlite3_stmt **ppStmt /* OUT: Compiled statement */
){
int rc;
@@ -464,14 +597,16 @@ int sqlite3Fts3AllSegdirs(
/* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
- sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1);
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
+ sqlite3_bind_int64(pStmt, 2,
+ getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
+ );
}
}else{
/* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL);
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel));
}
}
*ppStmt = pStmt;
@@ -637,6 +772,7 @@ static int fts3PendingTermsAddOne(
*/
static int fts3PendingTermsAdd(
Fts3Table *p, /* Table into which text will be inserted */
+ int iLangid, /* Language id to use */
const char *zText, /* Text of document to be inserted */
int iCol, /* Column into which text is being inserted */
u32 *pnWord /* OUT: Number of tokens inserted */
@@ -666,11 +802,10 @@ static int fts3PendingTermsAdd(
return SQLITE_OK;
}
- rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
+ rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr);
if( rc!=SQLITE_OK ){
return rc;
}
- pCsr->pTokenizer = pTokenizer;
xNext = pModule->xNext;
while( SQLITE_OK==rc
@@ -713,18 +848,28 @@ static int fts3PendingTermsAdd(
** fts3PendingTermsAdd() are to add term/position-list pairs for the
** contents of the document with docid iDocid.
*/
-static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
+static int fts3PendingTermsDocid(
+ Fts3Table *p, /* Full-text table handle */
+ int iLangid, /* Language id of row being written */
+ sqlite_int64 iDocid /* Docid of row being written */
+){
+ assert( iLangid>=0 );
+
/* TODO(shess) Explore whether partially flushing the buffer on
** forced-flush would provide better performance. I suspect that if
** we ordered the doclists by size and flushed the largest until the
** buffer was half empty, that would let the less frequent terms
** generate longer doclists.
*/
- if( iDocid<=p->iPrevDocid || p->nPendingData>p->nMaxPendingData ){
+ if( iDocid<=p->iPrevDocid
+ || p->iPrevLangid!=iLangid
+ || p->nPendingData>p->nMaxPendingData
+ ){
int rc = sqlite3Fts3PendingTermsFlush(p);
if( rc!=SQLITE_OK ) return rc;
}
p->iPrevDocid = iDocid;
+ p->iPrevLangid = iLangid;
return SQLITE_OK;
}
@@ -753,11 +898,16 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *p){
** Argument apVal is the same as the similarly named argument passed to
** fts3InsertData(). Parameter iDocid is the docid of the new row.
*/
-static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
+static int fts3InsertTerms(
+ Fts3Table *p,
+ int iLangid,
+ sqlite3_value **apVal,
+ u32 *aSz
+){
int i; /* Iterator variable */
for(i=2; i<p->nColumn+2; i++){
const char *zText = (const char *)sqlite3_value_text(apVal[i]);
- int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
+ int rc = fts3PendingTermsAdd(p, iLangid, zText, i-2, &aSz[i-2]);
if( rc!=SQLITE_OK ){
return rc;
}
@@ -778,6 +928,7 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
** apVal[p->nColumn+1] Right-most user-defined column
** apVal[p->nColumn+2] Hidden column with same name as table
** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid)
+** apVal[p->nColumn+4] Hidden languageid column
*/
static int fts3InsertData(
Fts3Table *p, /* Full-text table */
@@ -808,9 +959,13 @@ static int fts3InsertData(
** defined columns in the FTS3 table, plus one for the docid field.
*/
rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
- if( rc!=SQLITE_OK ){
- return rc;
+ if( rc==SQLITE_OK && p->zLanguageid ){
+ rc = sqlite3_bind_int(
+ pContentInsert, p->nColumn+2,
+ sqlite3_value_int(apVal[p->nColumn+4])
+ );
}
+ if( rc!=SQLITE_OK ) return rc;
/* There is a quirk here. The users INSERT statement may have specified
** a value for the "rowid" field, for the "docid" field, or for both.
@@ -871,6 +1026,15 @@ static int fts3DeleteAll(Fts3Table *p, int bContent){
}
/*
+**
+*/
+static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){
+ int iLangid = 0;
+ if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1);
+ return iLangid;
+}
+
+/*
** The first element in the apVal[] array is assumed to contain the docid
** (an integer) of a row about to be deleted. Remove all terms from the
** full-text index.
@@ -889,16 +1053,18 @@ static void fts3DeleteTerms(
if( rc==SQLITE_OK ){
if( SQLITE_ROW==sqlite3_step(pSelect) ){
int i;
- for(i=1; i<=p->nColumn; i++){
+ int iLangid = langidFromSelect(p, pSelect);
+ rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0));
+ for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){
const char *zText = (const char *)sqlite3_column_text(pSelect, i);
- rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]);
- if( rc!=SQLITE_OK ){
- sqlite3_reset(pSelect);
- *pRC = rc;
- return;
- }
+ rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]);
aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
}
+ if( rc!=SQLITE_OK ){
+ sqlite3_reset(pSelect);
+ *pRC = rc;
+ return;
+ }
}
rc = sqlite3_reset(pSelect);
}else{
@@ -911,7 +1077,7 @@ static void fts3DeleteTerms(
** Forward declaration to account for the circular dependency between
** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
*/
-static int fts3SegmentMerge(Fts3Table *, int, int);
+static int fts3SegmentMerge(Fts3Table *, int, int, int);
/*
** This function allocates a new level iLevel index in the segdir table.
@@ -930,6 +1096,7 @@ static int fts3SegmentMerge(Fts3Table *, int, int);
*/
static int fts3AllocateSegdirIdx(
Fts3Table *p,
+ int iLangid, /* Language id */
int iIndex, /* Index for p->aIndex */
int iLevel,
int *piIdx
@@ -938,10 +1105,15 @@ static int fts3AllocateSegdirIdx(
sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */
int iNext = 0; /* Result of query pNextIdx */
+ assert( iLangid>=0 );
+ assert( p->nIndex>=1 );
+
/* Set variable iNext to the next available segdir index at level iLevel. */
rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
+ sqlite3_bind_int64(
+ pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)
+ );
if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
iNext = sqlite3_column_int(pNextIdx, 0);
}
@@ -955,7 +1127,8 @@ static int fts3AllocateSegdirIdx(
** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
*/
if( iNext>=FTS3_MERGE_COUNT ){
- rc = fts3SegmentMerge(p, iIndex, iLevel);
+ fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
+ rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel);
*piIdx = 0;
}else{
*piIdx = iNext;
@@ -1002,7 +1175,7 @@ int sqlite3Fts3ReadBlock(
int rc; /* Return code */
/* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
- assert( pnBlob);
+ assert( pnBlob );
if( p->pSegments ){
rc = sqlite3_blob_reopen(p->pSegments, iBlockid);
@@ -1089,6 +1262,18 @@ static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
}
/*
+** Set an Fts3SegReader cursor to point at EOF.
+*/
+static void fts3SegReaderSetEof(Fts3SegReader *pSeg){
+ if( !fts3SegReaderIsRootOnly(pSeg) ){
+ sqlite3_free(pSeg->aNode);
+ sqlite3_blob_close(pSeg->pBlob);
+ pSeg->pBlob = 0;
+ }
+ pSeg->aNode = 0;
+}
+
+/*
** Move the iterator passed as the first argument to the next term in the
** segment. If successful, SQLITE_OK is returned. If there is no next term,
** SQLITE_DONE. Otherwise, an SQLite error code.
@@ -1127,12 +1312,7 @@ static int fts3SegReaderNext(
return SQLITE_OK;
}
- if( !fts3SegReaderIsRootOnly(pReader) ){
- sqlite3_free(pReader->aNode);
- sqlite3_blob_close(pReader->pBlob);
- pReader->pBlob = 0;
- }
- pReader->aNode = 0;
+ fts3SegReaderSetEof(pReader);
/* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
** blocks have already been traversed. */
@@ -1336,7 +1516,7 @@ int sqlite3Fts3MsrOvfl(
int rc = SQLITE_OK;
int pgsz = p->nPgsz;
- assert( p->bHasStat );
+ assert( p->bFts4 );
assert( pgsz>0 );
for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
@@ -1379,6 +1559,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
*/
int sqlite3Fts3SegReaderNew(
int iAge, /* Segment "age". */
+ int bLookup, /* True for a lookup only */
sqlite3_int64 iStartLeaf, /* First leaf to traverse */
sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
sqlite3_int64 iEndBlock, /* Final block of segment */
@@ -1386,7 +1567,6 @@ int sqlite3Fts3SegReaderNew(
int nRoot, /* Size of buffer containing root node */
Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
){
- int rc = SQLITE_OK; /* Return code */
Fts3SegReader *pReader; /* Newly allocated SegReader object */
int nExtra = 0; /* Bytes to allocate segment root node */
@@ -1401,6 +1581,7 @@ int sqlite3Fts3SegReaderNew(
}
memset(pReader, 0, sizeof(Fts3SegReader));
pReader->iIdx = iAge;
+ pReader->bLookup = bLookup;
pReader->iStartBlock = iStartLeaf;
pReader->iLeafEndBlock = iEndLeaf;
pReader->iEndBlock = iEndBlock;
@@ -1414,13 +1595,8 @@ int sqlite3Fts3SegReaderNew(
}else{
pReader->iCurrentBlock = iStartLeaf-1;
}
-
- if( rc==SQLITE_OK ){
- *ppReader = pReader;
- }else{
- sqlite3Fts3SegReaderFree(pReader);
- }
- return rc;
+ *ppReader = pReader;
+ return SQLITE_OK;
}
/*
@@ -1470,6 +1646,7 @@ int sqlite3Fts3SegReaderPending(
Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */
){
Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */
+ Fts3HashElem *pE; /* Iterator variable */
Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */
int nElem = 0; /* Size of array at aElem */
int rc = SQLITE_OK; /* Return Code */
@@ -1478,7 +1655,6 @@ int sqlite3Fts3SegReaderPending(
pHash = &p->aIndex[iIndex].hPending;
if( bPrefix ){
int nAlloc = 0; /* Size of allocated array at aElem */
- Fts3HashElem *pE = 0; /* Iterator variable */
for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){
char *zKey = (char *)fts3HashKey(pE);
@@ -1512,8 +1688,13 @@ int sqlite3Fts3SegReaderPending(
}else{
/* The query is a simple term lookup that matches at most one term in
- ** the index. All that is required is a straight hash-lookup. */
- Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm);
+ ** the index. All that is required is a straight hash-lookup.
+ **
+ ** Because the stack address of pE may be accessed via the aElem pointer
+ ** below, the "Fts3HashElem *pE" must be declared so that it is valid
+ ** within this entire function, not just this "else{...}" block.
+ */
+ pE = fts3HashFindElem(pHash, zTerm, nTerm);
if( pE ){
aElem = &pE;
nElem = 1;
@@ -1693,12 +1874,33 @@ static int fts3WriteSegment(
return rc;
}
+/*
+** Find the largest relative level number in the table. If successful, set
+** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs,
+** set *pnMax to zero and return an SQLite error code.
+*/
+int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){
+ int rc;
+ int mxLevel = 0;
+ sqlite3_stmt *pStmt = 0;
+
+ rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ mxLevel = sqlite3_column_int(pStmt, 0);
+ }
+ rc = sqlite3_reset(pStmt);
+ }
+ *pnMax = mxLevel;
+ return rc;
+}
+
/*
** Insert a record into the %_segdir table.
*/
static int fts3WriteSegdir(
Fts3Table *p, /* Virtual table handle */
- int iLevel, /* Value for "level" field */
+ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */
int iIdx, /* Value for "idx" field */
sqlite3_int64 iStartBlock, /* Value for "start_block" field */
sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
@@ -1709,7 +1911,7 @@ static int fts3WriteSegdir(
sqlite3_stmt *pStmt;
int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pStmt, 1, iLevel);
+ sqlite3_bind_int64(pStmt, 1, iLevel);
sqlite3_bind_int(pStmt, 2, iIdx);
sqlite3_bind_int64(pStmt, 3, iStartBlock);
sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
@@ -2009,6 +2211,7 @@ static int fts3SegWriterAdd(
/* The current leaf node is full. Write it out to the database. */
rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
if( rc!=SQLITE_OK ) return rc;
+ p->nLeafAdd++;
/* Add the current term to the interior node tree. The term added to
** the interior tree must:
@@ -2092,7 +2295,7 @@ static int fts3SegWriterAdd(
static int fts3SegWriterFlush(
Fts3Table *p, /* Virtual table handle */
SegmentWriter *pWriter, /* SegmentWriter to flush to the db */
- int iLevel, /* Value for 'level' column of %_segdir */
+ sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */
int iIdx /* Value for 'idx' column of %_segdir */
){
int rc; /* Return code */
@@ -2117,6 +2320,7 @@ static int fts3SegWriterFlush(
rc = fts3WriteSegdir(
p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
}
+ p->nLeafAdd++;
return rc;
}
@@ -2170,7 +2374,12 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
**
** Return SQLITE_OK if successful, or an SQLite error code if not.
*/
-static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
+static int fts3SegmentMaxLevel(
+ Fts3Table *p,
+ int iLangid,
+ int iIndex,
+ sqlite3_int64 *pnMax
+){
sqlite3_stmt *pStmt;
int rc;
assert( iIndex>=0 && iIndex<p->nIndex );
@@ -2183,15 +2392,40 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
*/
rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
if( rc!=SQLITE_OK ) return rc;
- sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
- sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1);
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
+ sqlite3_bind_int64(pStmt, 2,
+ getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
+ );
if( SQLITE_ROW==sqlite3_step(pStmt) ){
- *pnMax = sqlite3_column_int(pStmt, 0);
+ *pnMax = sqlite3_column_int64(pStmt, 0);
}
return sqlite3_reset(pStmt);
}
/*
+** Delete all entries in the %_segments table associated with the segment
+** opened with seg-reader pSeg. This function does not affect the contents
+** of the %_segdir table.
+*/
+static int fts3DeleteSegment(
+ Fts3Table *p, /* FTS table handle */
+ Fts3SegReader *pSeg /* Segment to delete */
+){
+ int rc = SQLITE_OK; /* Return code */
+ if( pSeg->iStartBlock ){
+ sqlite3_stmt *pDelete; /* SQL statement to delete rows */
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock);
+ sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock);
+ sqlite3_step(pDelete);
+ rc = sqlite3_reset(pDelete);
+ }
+ }
+ return rc;
+}
+
+/*
** This function is used after merging multiple segments into a single large
** segment to delete the old, now redundant, segment b-trees. Specifically,
** it:
@@ -2207,24 +2441,18 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
*/
static int fts3DeleteSegdir(
Fts3Table *p, /* Virtual table handle */
+ int iLangid, /* Language id */
int iIndex, /* Index for p->aIndex */
int iLevel, /* Level of %_segdir entries to delete */
Fts3SegReader **apSegment, /* Array of SegReader objects */
int nReader /* Size of array apSegment */
){
- int rc; /* Return Code */
+ int rc = SQLITE_OK; /* Return Code */
int i; /* Iterator variable */
- sqlite3_stmt *pDelete; /* SQL statement to delete rows */
+ sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */
- rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
for(i=0; rc==SQLITE_OK && i<nReader; i++){
- Fts3SegReader *pSegment = apSegment[i];
- if( pSegment->iStartBlock ){
- sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock);
- sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock);
- sqlite3_step(pDelete);
- rc = sqlite3_reset(pDelete);
- }
+ rc = fts3DeleteSegment(p, apSegment[i]);
}
if( rc!=SQLITE_OK ){
return rc;
@@ -2234,13 +2462,17 @@ static int fts3DeleteSegdir(
if( iLevel==FTS3_SEGCURSOR_ALL ){
rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
- sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1);
+ sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
+ sqlite3_bind_int64(pDelete, 2,
+ getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
+ );
}
}else{
rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
+ sqlite3_bind_int64(
+ pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)
+ );
}
}
@@ -2403,11 +2635,16 @@ static int fts3SegReaderStart(
** b-tree leaf nodes contain more than one term.
*/
for(i=0; pCsr->bRestart==0 && i<pCsr->nSegment; i++){
+ int res = 0;
Fts3SegReader *pSeg = pCsr->apSegment[i];
do {
int rc = fts3SegReaderNext(p, pSeg, 0);
if( rc!=SQLITE_OK ) return rc;
- }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
+ }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 );
+
+ if( pSeg->bLookup && res!=0 ){
+ fts3SegReaderSetEof(pSeg);
+ }
}
fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp);
@@ -2528,7 +2765,12 @@ int sqlite3Fts3SegReaderStep(
** forward. Then sort the list in order of current term again.
*/
for(i=0; i<pCsr->nAdvance; i++){
- rc = fts3SegReaderNext(p, apSegment[i], 0);
+ Fts3SegReader *pSeg = apSegment[i];
+ if( pSeg->bLookup ){
+ fts3SegReaderSetEof(pSeg);
+ }else{
+ rc = fts3SegReaderNext(p, pSeg, 0);
+ }
if( rc!=SQLITE_OK ) return rc;
}
fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
@@ -2699,13 +2941,18 @@ void sqlite3Fts3SegReaderFinish(
** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
** an SQLite error code is returned.
*/
-static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
+static int fts3SegmentMerge(
+ Fts3Table *p,
+ int iLangid, /* Language id to merge */
+ int iIndex, /* Index in p->aIndex[] to merge */
+ int iLevel /* Level to merge */
+){
int rc; /* Return code */
int iIdx = 0; /* Index of new segment */
- int iNewLevel = 0; /* Level/index to create new segment at */
+ sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */
SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
Fts3SegFilter filter; /* Segment term filter condition */
- Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
+ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
int bIgnoreEmpty = 0; /* True to ignore empty segments */
assert( iLevel==FTS3_SEGCURSOR_ALL
@@ -2715,7 +2962,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
assert( iIndex>=0 && iIndex<p->nIndex );
- rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr);
+ rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr);
if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
if( iLevel==FTS3_SEGCURSOR_ALL ){
@@ -2727,24 +2974,24 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
rc = SQLITE_DONE;
goto finished;
}
- rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel);
+ rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel);
bIgnoreEmpty = 1;
}else if( iLevel==FTS3_SEGCURSOR_PENDING ){
- iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL;
- rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx);
+ iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0);
+ rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx);
}else{
/* This call is to merge all segments at level iLevel. find the next
** available segment index at level iLevel+1. The call to
** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
** a single iLevel+2 segment if necessary. */
- rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx);
- iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1;
+ rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx);
+ iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1);
}
if( rc!=SQLITE_OK ) goto finished;
assert( csr.nSegment>0 );
- assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) );
- assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) );
+ assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
+ assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) );
memset(&filter, 0, sizeof(Fts3SegFilter));
filter.flags = FTS3_SEGMENT_REQUIRE_POS;
@@ -2761,7 +3008,9 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
assert( pWriter );
if( iLevel!=FTS3_SEGCURSOR_PENDING ){
- rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment);
+ rc = fts3DeleteSegdir(
+ p, iLangid, iIndex, iLevel, csr.apSegment, csr.nSegment
+ );
if( rc!=SQLITE_OK ) goto finished;
}
rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
@@ -2779,11 +3028,28 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
int rc = SQLITE_OK;
int i;
+
for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
- rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING);
+ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
if( rc==SQLITE_DONE ) rc = SQLITE_OK;
}
sqlite3Fts3PendingTermsClear(p);
+
+ /* Determine the auto-incr-merge setting if unknown. If enabled,
+ ** estimate the number of leaf blocks of content to be written
+ */
+ if( rc==SQLITE_OK && p->bHasStat
+ && p->bAutoincrmerge==0xff && p->nLeafAdd>0
+ ){
+ sqlite3_stmt *pStmt = 0;
+ rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
+ rc = sqlite3_step(pStmt);
+ p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0));
+ rc = sqlite3_reset(pStmt);
+ }
+ }
return rc;
}
@@ -2894,12 +3160,13 @@ static void fts3UpdateDocTotals(
return;
}
pBlob = (char*)&a[nStat];
- rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
+ rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0);
if( rc ){
sqlite3_free(a);
*pRC = rc;
return;
}
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
if( sqlite3_step(pStmt)==SQLITE_ROW ){
fts3DecodeIntArray(nStat, a,
sqlite3_column_blob(pStmt, 0),
@@ -2923,29 +3190,47 @@ static void fts3UpdateDocTotals(
a[i+1] = x;
}
fts3EncodeIntArray(nStat, a, pBlob, &nBlob);
- rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0);
+ rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
if( rc ){
sqlite3_free(a);
*pRC = rc;
return;
}
- sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
+ sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
sqlite3_step(pStmt);
*pRC = sqlite3_reset(pStmt);
sqlite3_free(a);
}
+/*
+** Merge the entire database so that there is one segment for each
+** iIndex/iLangid combination.
+*/
static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
- int i;
int bSeenDone = 0;
- int rc = SQLITE_OK;
- for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
- rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL);
- if( rc==SQLITE_DONE ){
- bSeenDone = 1;
- rc = SQLITE_OK;
+ int rc;
+ sqlite3_stmt *pAllLangid = 0;
+
+ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
+ if( rc==SQLITE_OK ){
+ int rc2;
+ sqlite3_bind_int(pAllLangid, 1, p->nIndex);
+ while( sqlite3_step(pAllLangid)==SQLITE_ROW ){
+ int i;
+ int iLangid = sqlite3_column_int(pAllLangid, 0);
+ for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
+ rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL);
+ if( rc==SQLITE_DONE ){
+ bSeenDone = 1;
+ rc = SQLITE_OK;
+ }
+ }
}
+ rc2 = sqlite3_reset(pAllLangid);
+ if( rc==SQLITE_OK ) rc = rc2;
}
+
sqlite3Fts3SegmentsClose(p);
sqlite3Fts3PendingTermsClear(p);
@@ -2996,11 +3281,12 @@ static int fts3DoRebuild(Fts3Table *p){
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
int iCol;
- rc = fts3PendingTermsDocid(p, sqlite3_column_int64(pStmt, 0));
+ int iLangid = langidFromSelect(p, pStmt);
+ rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0));
aSz[p->nColumn] = 0;
for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
- rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]);
+ rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]);
aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
}
if( p->bHasDocsize ){
@@ -3016,7 +3302,7 @@ static int fts3DoRebuild(Fts3Table *p){
}
}
}
- if( p->bHasStat ){
+ if( p->bFts4 ){
fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry);
}
sqlite3_free(aSz);
@@ -3032,6 +3318,1687 @@ static int fts3DoRebuild(Fts3Table *p){
return rc;
}
+
+/*
+** This function opens a cursor used to read the input data for an
+** incremental merge operation. Specifically, it opens a cursor to scan
+** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute
+** level iAbsLevel.
+*/
+static int fts3IncrmergeCsr(
+ Fts3Table *p, /* FTS3 table handle */
+ sqlite3_int64 iAbsLevel, /* Absolute level to open */
+ int nSeg, /* Number of segments to merge */
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */
+){
+ int rc; /* Return Code */
+ sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */
+ int nByte; /* Bytes allocated at pCsr->apSegment[] */
+
+ /* Allocate space for the Fts3MultiSegReader.aCsr[] array */
+ memset(pCsr, 0, sizeof(*pCsr));
+ nByte = sizeof(Fts3SegReader *) * nSeg;
+ pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
+
+ if( pCsr->apSegment==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(pCsr->apSegment, 0, nByte);
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
+ }
+ if( rc==SQLITE_OK ){
+ int i;
+ int rc2;
+ sqlite3_bind_int64(pStmt, 1, iAbsLevel);
+ assert( pCsr->nSegment==0 );
+ for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && i<nSeg; i++){
+ rc = sqlite3Fts3SegReaderNew(i, 0,
+ sqlite3_column_int64(pStmt, 1), /* segdir.start_block */
+ sqlite3_column_int64(pStmt, 2), /* segdir.leaves_end_block */
+ sqlite3_column_int64(pStmt, 3), /* segdir.end_block */
+ sqlite3_column_blob(pStmt, 4), /* segdir.root */
+ sqlite3_column_bytes(pStmt, 4), /* segdir.root */
+ &pCsr->apSegment[i]
+ );
+ pCsr->nSegment++;
+ }
+ rc2 = sqlite3_reset(pStmt);
+ if( rc==SQLITE_OK ) rc = rc2;
+ }
+
+ return rc;
+}
+
+typedef struct IncrmergeWriter IncrmergeWriter;
+typedef struct NodeWriter NodeWriter;
+typedef struct Blob Blob;
+typedef struct NodeReader NodeReader;
+
+/*
+** An instance of the following structure is used as a dynamic buffer
+** to build up nodes or other blobs of data in.
+**
+** The function blobGrowBuffer() is used to extend the allocation.
+*/
+struct Blob {
+ char *a; /* Pointer to allocation */
+ int n; /* Number of valid bytes of data in a[] */
+ int nAlloc; /* Allocated size of a[] (nAlloc>=n) */
+};
+
+/*
+** This structure is used to build up buffers containing segment b-tree
+** nodes (blocks).
+*/
+struct NodeWriter {
+ sqlite3_int64 iBlock; /* Current block id */
+ Blob key; /* Last key written to the current block */
+ Blob block; /* Current block image */
+};
+
+/*
+** An object of this type contains the state required to create or append
+** to an appendable b-tree segment.
+*/
+struct IncrmergeWriter {
+ int nLeafEst; /* Space allocated for leaf blocks */
+ int nWork; /* Number of leaf pages flushed */
+ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */
+ int iIdx; /* Index of *output* segment in iAbsLevel+1 */
+ sqlite3_int64 iStart; /* Block number of first allocated block */
+ sqlite3_int64 iEnd; /* Block number of last allocated block */
+ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT];
+};
+
+/*
+** An object of the following type is used to read data from a single
+** FTS segment node. See the following functions:
+**
+** nodeReaderInit()
+** nodeReaderNext()
+** nodeReaderRelease()
+*/
+struct NodeReader {
+ const char *aNode;
+ int nNode;
+ int iOff; /* Current offset within aNode[] */
+
+ /* Output variables. Containing the current node entry. */
+ sqlite3_int64 iChild; /* Pointer to child node */
+ Blob term; /* Current term */
+ const char *aDoclist; /* Pointer to doclist */
+ int nDoclist; /* Size of doclist in bytes */
+};
+
+/*
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
+** Otherwise, if the allocation at pBlob->a is not already at least nMin
+** bytes in size, extend (realloc) it to be so.
+**
+** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a
+** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc
+** to reflect the new size of the pBlob->a[] buffer.
+*/
+static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){
+ if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){
+ int nAlloc = nMin;
+ char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc);
+ if( a ){
+ pBlob->nAlloc = nAlloc;
+ pBlob->a = a;
+ }else{
+ *pRc = SQLITE_NOMEM;
+ }
+ }
+}
+
+/*
+** Attempt to advance the node-reader object passed as the first argument to
+** the next entry on the node.
+**
+** Return an error code if an error occurs (SQLITE_NOMEM is possible).
+** Otherwise return SQLITE_OK. If there is no next entry on the node
+** (e.g. because the current entry is the last) set NodeReader->aNode to
+** NULL to indicate EOF. Otherwise, populate the NodeReader structure output
+** variables for the new entry.
+*/
+static int nodeReaderNext(NodeReader *p){
+ int bFirst = (p->term.n==0); /* True for first term on the node */
+ int nPrefix = 0; /* Bytes to copy from previous term */
+ int nSuffix = 0; /* Bytes to append to the prefix */
+ int rc = SQLITE_OK; /* Return code */
+
+ assert( p->aNode );
+ if( p->iChild && bFirst==0 ) p->iChild++;
+ if( p->iOff>=p->nNode ){
+ /* EOF */
+ p->aNode = 0;
+ }else{
+ if( bFirst==0 ){
+ p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
+ }
+ p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
+
+ blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
+ if( rc==SQLITE_OK ){
+ memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
+ p->term.n = nPrefix+nSuffix;
+ p->iOff += nSuffix;
+ if( p->iChild==0 ){
+ p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
+ p->aDoclist = &p->aNode[p->iOff];
+ p->iOff += p->nDoclist;
+ }
+ }
+ }
+
+ assert( p->iOff<=p->nNode );
+
+ return rc;
+}
+
+/*
+** Release all dynamic resources held by node-reader object *p.
+*/
+static void nodeReaderRelease(NodeReader *p){
+ sqlite3_free(p->term.a);
+}
+
+/*
+** Initialize a node-reader object to read the node in buffer aNode/nNode.
+**
+** If successful, SQLITE_OK is returned and the NodeReader object set to
+** point to the first entry on the node (if any). Otherwise, an SQLite
+** error code is returned.
+*/
+static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){
+ memset(p, 0, sizeof(NodeReader));
+ p->aNode = aNode;
+ p->nNode = nNode;
+
+ /* Figure out if this is a leaf or an internal node. */
+ if( p->aNode[0] ){
+ /* An internal node. */
+ p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild);
+ }else{
+ p->iOff = 1;
+ }
+
+ return nodeReaderNext(p);
+}
+
+/*
+** This function is called while writing an FTS segment each time a leaf o
+** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed
+** to be greater than the largest key on the node just written, but smaller
+** than or equal to the first key that will be written to the next leaf
+** node.
+**
+** The block id of the leaf node just written to disk may be found in
+** (pWriter->aNodeWriter[0].iBlock) when this function is called.
+*/
+static int fts3IncrmergePush(
+ Fts3Table *p, /* Fts3 table handle */
+ IncrmergeWriter *pWriter, /* Writer object */
+ const char *zTerm, /* Term to write to internal node */
+ int nTerm /* Bytes at zTerm */
+){
+ sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock;
+ int iLayer;
+
+ assert( nTerm>0 );
+ for(iLayer=1; ALWAYS(iLayer<FTS_MAX_APPENDABLE_HEIGHT); iLayer++){
+ sqlite3_int64 iNextPtr = 0;
+ NodeWriter *pNode = &pWriter->aNodeWriter[iLayer];
+ int rc = SQLITE_OK;
+ int nPrefix;
+ int nSuffix;
+ int nSpace;
+
+ /* Figure out how much space the key will consume if it is written to
+ ** the current node of layer iLayer. Due to the prefix compression,
+ ** the space required changes depending on which node the key is to
+ ** be added to. */
+ nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm);
+ nSuffix = nTerm - nPrefix;
+ nSpace = sqlite3Fts3VarintLen(nPrefix);
+ nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;
+
+ if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){
+ /* If the current node of layer iLayer contains zero keys, or if adding
+ ** the key to it will not cause it to grow to larger than nNodeSize
+ ** bytes in size, write the key here. */
+
+ Blob *pBlk = &pNode->block;
+ if( pBlk->n==0 ){
+ blobGrowBuffer(pBlk, p->nNodeSize, &rc);
+ if( rc==SQLITE_OK ){
+ pBlk->a[0] = (char)iLayer;
+ pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr);
+ }
+ }
+ blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc);
+ blobGrowBuffer(&pNode->key, nTerm, &rc);
+
+ if( rc==SQLITE_OK ){
+ if( pNode->key.n ){
+ pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix);
+ }
+ pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix);
+ memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix);
+ pBlk->n += nSuffix;
+
+ memcpy(pNode->key.a, zTerm, nTerm);
+ pNode->key.n = nTerm;
+ }
+ }else{
+ /* Otherwise, flush the the current node of layer iLayer to disk.
+ ** Then allocate a new, empty sibling node. The key will be written
+ ** into the parent of this node. */
+ rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n);
+
+ assert( pNode->block.nAlloc>=p->nNodeSize );
+ pNode->block.a[0] = (char)iLayer;
+ pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1);
+
+ iNextPtr = pNode->iBlock;
+ pNode->iBlock++;
+ pNode->key.n = 0;
+ }
+
+ if( rc!=SQLITE_OK || iNextPtr==0 ) return rc;
+ iPtr = iNextPtr;
+ }
+
+ assert( 0 );
+ return 0;
+}
+
+/*
+** Append a term and (optionally) doclist to the FTS segment node currently
+** stored in blob *pNode. The node need not contain any terms, but the
+** header must be written before this function is called.
+**
+** A node header is a single 0x00 byte for a leaf node, or a height varint
+** followed by the left-hand-child varint for an internal node.
+**
+** The term to be appended is passed via arguments zTerm/nTerm. For a
+** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal
+** node, both aDoclist and nDoclist must be passed 0.
+**
+** If the size of the value in blob pPrev is zero, then this is the first
+** term written to the node. Otherwise, pPrev contains a copy of the
+** previous term. Before this function returns, it is updated to contain a
+** copy of zTerm/nTerm.
+**
+** It is assumed that the buffer associated with pNode is already large
+** enough to accommodate the new entry. The buffer associated with pPrev
+** is extended by this function if requrired.
+**
+** If an error (i.e. OOM condition) occurs, an SQLite error code is
+** returned. Otherwise, SQLITE_OK.
+*/
+static int fts3AppendToNode(
+ Blob *pNode, /* Current node image to append to */
+ Blob *pPrev, /* Buffer containing previous term written */
+ const char *zTerm, /* New term to write */
+ int nTerm, /* Size of zTerm in bytes */
+ const char *aDoclist, /* Doclist (or NULL) to write */
+ int nDoclist /* Size of aDoclist in bytes */
+){
+ int rc = SQLITE_OK; /* Return code */
+ int bFirst = (pPrev->n==0); /* True if this is the first term written */
+ int nPrefix; /* Size of term prefix in bytes */
+ int nSuffix; /* Size of term suffix in bytes */
+
+ /* Node must have already been started. There must be a doclist for a
+ ** leaf node, and there must not be a doclist for an internal node. */
+ assert( pNode->n>0 );
+ assert( (pNode->a[0]=='\0')==(aDoclist!=0) );
+
+ blobGrowBuffer(pPrev, nTerm, &rc);
+ if( rc!=SQLITE_OK ) return rc;
+
+ nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm);
+ nSuffix = nTerm - nPrefix;
+ memcpy(pPrev->a, zTerm, nTerm);
+ pPrev->n = nTerm;
+
+ if( bFirst==0 ){
+ pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix);
+ }
+ pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix);
+ memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix);
+ pNode->n += nSuffix;
+
+ if( aDoclist ){
+ pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist);
+ memcpy(&pNode->a[pNode->n], aDoclist, nDoclist);
+ pNode->n += nDoclist;
+ }
+
+ assert( pNode->n<=pNode->nAlloc );
+
+ return SQLITE_OK;
+}
+
+/*
+** Append the current term and doclist pointed to by cursor pCsr to the
+** appendable b-tree segment opened for writing by pWriter.
+**
+** Return SQLITE_OK if successful, or an SQLite error code otherwise.
+*/
+static int fts3IncrmergeAppend(
+ Fts3Table *p, /* Fts3 table handle */
+ IncrmergeWriter *pWriter, /* Writer object */
+ Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */
+){
+ const char *zTerm = pCsr->zTerm;
+ int nTerm = pCsr->nTerm;
+ const char *aDoclist = pCsr->aDoclist;
+ int nDoclist = pCsr->nDoclist;
+ int rc = SQLITE_OK; /* Return code */
+ int nSpace; /* Total space in bytes required on leaf */
+ int nPrefix; /* Size of prefix shared with previous term */
+ int nSuffix; /* Size of suffix (nTerm - nPrefix) */
+ NodeWriter *pLeaf; /* Object used to write leaf nodes */
+
+ pLeaf = &pWriter->aNodeWriter[0];
+ nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm);
+ nSuffix = nTerm - nPrefix;
+
+ nSpace = sqlite3Fts3VarintLen(nPrefix);
+ nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;
+ nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
+
+ /* If the current block is not empty, and if adding this term/doclist
+ ** to the current block would make it larger than Fts3Table.nNodeSize
+ ** bytes, write this block out to the database. */
+ if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){
+ rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n);
+ pWriter->nWork++;
+
+ /* Add the current term to the parent node. The term added to the
+ ** parent must:
+ **
+ ** a) be greater than the largest term on the leaf node just written
+ ** to the database (still available in pLeaf->key), and
+ **
+ ** b) be less than or equal to the term about to be added to the new
+ ** leaf node (zTerm/nTerm).
+ **
+ ** In other words, it must be the prefix of zTerm 1 byte longer than
+ ** the common prefix (if any) of zTerm and pWriter->zTerm.
+ */
+ if( rc==SQLITE_OK ){
+ rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1);
+ }
+
+ /* Advance to the next output block */
+ pLeaf->iBlock++;
+ pLeaf->key.n = 0;
+ pLeaf->block.n = 0;
+
+ nSuffix = nTerm;
+ nSpace = 1;
+ nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;
+ nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
+ }
+
+ blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc);
+
+ if( rc==SQLITE_OK ){
+ if( pLeaf->block.n==0 ){
+ pLeaf->block.n = 1;
+ pLeaf->block.a[0] = '\0';
+ }
+ rc = fts3AppendToNode(
+ &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist
+ );
+ }
+
+ return rc;
+}
+
+/*
+** This function is called to release all dynamic resources held by the
+** merge-writer object pWriter, and if no error has occurred, to flush
+** all outstanding node buffers held by pWriter to disk.
+**
+** If *pRc is not SQLITE_OK when this function is called, then no attempt
+** is made to write any data to disk. Instead, this function serves only
+** to release outstanding resources.
+**
+** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while
+** flushing buffers to disk, *pRc is set to an SQLite error code before
+** returning.
+*/
+static void fts3IncrmergeRelease(
+ Fts3Table *p, /* FTS3 table handle */
+ IncrmergeWriter *pWriter, /* Merge-writer object */
+ int *pRc /* IN/OUT: Error code */
+){
+ int i; /* Used to iterate through non-root layers */
+ int iRoot; /* Index of root in pWriter->aNodeWriter */
+ NodeWriter *pRoot; /* NodeWriter for root node */
+ int rc = *pRc; /* Error code */
+
+ /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment
+ ** root node. If the segment fits entirely on a single leaf node, iRoot
+ ** will be set to 0. If the root node is the parent of the leaves, iRoot
+ ** will be 1. And so on. */
+ for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){
+ NodeWriter *pNode = &pWriter->aNodeWriter[iRoot];
+ if( pNode->block.n>0 ) break;
+ assert( *pRc || pNode->block.nAlloc==0 );
+ assert( *pRc || pNode->key.nAlloc==0 );
+ sqlite3_free(pNode->block.a);
+ sqlite3_free(pNode->key.a);
+ }
+
+ /* Empty output segment. This is a no-op. */
+ if( iRoot<0 ) return;
+
+ /* The entire output segment fits on a single node. Normally, this means
+ ** the node would be stored as a blob in the "root" column of the %_segdir
+ ** table. However, this is not permitted in this case. The problem is that
+ ** space has already been reserved in the %_segments table, and so the
+ ** start_block and end_block fields of the %_segdir table must be populated.
+ ** And, by design or by accident, released versions of FTS cannot handle
+ ** segments that fit entirely on the root node with start_block!=0.
+ **
+ ** Instead, create a synthetic root node that contains nothing but a
+ ** pointer to the single content node. So that the segment consists of a
+ ** single leaf and a single interior (root) node.
+ **
+ ** Todo: Better might be to defer allocating space in the %_segments
+ ** table until we are sure it is needed.
+ */
+ if( iRoot==0 ){
+ Blob *pBlock = &pWriter->aNodeWriter[1].block;
+ blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc);
+ if( rc==SQLITE_OK ){
+ pBlock->a[0] = 0x01;
+ pBlock->n = 1 + sqlite3Fts3PutVarint(
+ &pBlock->a[1], pWriter->aNodeWriter[0].iBlock
+ );
+ }
+ iRoot = 1;
+ }
+ pRoot = &pWriter->aNodeWriter[iRoot];
+
+ /* Flush all currently outstanding nodes to disk. */
+ for(i=0; i<iRoot; i++){
+ NodeWriter *pNode = &pWriter->aNodeWriter[i];
+ if( pNode->block.n>0 && rc==SQLITE_OK ){
+ rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n);
+ }
+ sqlite3_free(pNode->block.a);
+ sqlite3_free(pNode->key.a);
+ }
+
+ /* Write the %_segdir record. */
+ if( rc==SQLITE_OK ){
+ rc = fts3WriteSegdir(p,
+ pWriter->iAbsLevel+1, /* level */
+ pWriter->iIdx, /* idx */
+ pWriter->iStart, /* start_block */
+ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */
+ pWriter->iEnd, /* end_block */
+ pRoot->block.a, pRoot->block.n /* root */
+ );
+ }
+ sqlite3_free(pRoot->block.a);
+ sqlite3_free(pRoot->key.a);
+
+ *pRc = rc;
+}
+
+/*
+** Compare the term in buffer zLhs (size in bytes nLhs) with that in
+** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of
+** the other, it is considered to be smaller than the other.
+**
+** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve
+** if it is greater.
+*/
+static int fts3TermCmp(
+ const char *zLhs, int nLhs, /* LHS of comparison */
+ const char *zRhs, int nRhs /* RHS of comparison */
+){
+ int nCmp = MIN(nLhs, nRhs);
+ int res;
+
+ res = memcmp(zLhs, zRhs, nCmp);
+ if( res==0 ) res = nLhs - nRhs;
+
+ return res;
+}
+
+
+/*
+** Query to see if the entry in the %_segments table with blockid iEnd is
+** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before
+** returning. Otherwise, set *pbRes to 0.
+**
+** Or, if an error occurs while querying the database, return an SQLite
+** error code. The final value of *pbRes is undefined in this case.
+**
+** This is used to test if a segment is an "appendable" segment. If it
+** is, then a NULL entry has been inserted into the %_segments table
+** with blockid %_segdir.end_block.
+*/
+static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){
+ int bRes = 0; /* Result to set *pbRes to */
+ sqlite3_stmt *pCheck = 0; /* Statement to query database with */
+ int rc; /* Return code */
+
+ rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pCheck, 1, iEnd);
+ if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1;
+ rc = sqlite3_reset(pCheck);
+ }
+
+ *pbRes = bRes;
+ return rc;
+}
+
+/*
+** This function is called when initializing an incremental-merge operation.
+** It checks if the existing segment with index value iIdx at absolute level
+** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the
+** merge-writer object *pWriter is initialized to write to it.
+**
+** An existing segment can be appended to by an incremental merge if:
+**
+** * It was initially created as an appendable segment (with all required
+** space pre-allocated), and
+**
+** * The first key read from the input (arguments zKey and nKey) is
+** greater than the largest key currently stored in the potential
+** output segment.
+*/
+static int fts3IncrmergeLoad(
+ Fts3Table *p, /* Fts3 table handle */
+ sqlite3_int64 iAbsLevel, /* Absolute level of input segments */
+ int iIdx, /* Index of candidate output segment */
+ const char *zKey, /* First key to write */
+ int nKey, /* Number of bytes in nKey */
+ IncrmergeWriter *pWriter /* Populate this object */
+){
+ int rc; /* Return code */
+ sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */
+
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */
+ sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */
+ sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */
+ const char *aRoot = 0; /* Pointer to %_segdir.root buffer */
+ int nRoot = 0; /* Size of aRoot[] in bytes */
+ int rc2; /* Return code from sqlite3_reset() */
+ int bAppendable = 0; /* Set to true if segment is appendable */
+
+ /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */
+ sqlite3_bind_int64(pSelect, 1, iAbsLevel+1);
+ sqlite3_bind_int(pSelect, 2, iIdx);
+ if( sqlite3_step(pSelect)==SQLITE_ROW ){
+ iStart = sqlite3_column_int64(pSelect, 1);
+ iLeafEnd = sqlite3_column_int64(pSelect, 2);
+ iEnd = sqlite3_column_int64(pSelect, 3);
+ nRoot = sqlite3_column_bytes(pSelect, 4);
+ aRoot = sqlite3_column_blob(pSelect, 4);
+ }else{
+ return sqlite3_reset(pSelect);
+ }
+
+ /* Check for the zero-length marker in the %_segments table */
+ rc = fts3IsAppendable(p, iEnd, &bAppendable);
+
+ /* Check that zKey/nKey is larger than the largest key the candidate */
+ if( rc==SQLITE_OK && bAppendable ){
+ char *aLeaf = 0;
+ int nLeaf = 0;
+
+ rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0);
+ if( rc==SQLITE_OK ){
+ NodeReader reader;
+ for(rc = nodeReaderInit(&reader, aLeaf, nLeaf);
+ rc==SQLITE_OK && reader.aNode;
+ rc = nodeReaderNext(&reader)
+ ){
+ assert( reader.aNode );
+ }
+ if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){
+ bAppendable = 0;
+ }
+ nodeReaderRelease(&reader);
+ }
+ sqlite3_free(aLeaf);
+ }
+
+ if( rc==SQLITE_OK && bAppendable ){
+ /* It is possible to append to this segment. Set up the IncrmergeWriter
+ ** object to do so. */
+ int i;
+ int nHeight = (int)aRoot[0];
+ NodeWriter *pNode;
+
+ pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT;
+ pWriter->iStart = iStart;
+ pWriter->iEnd = iEnd;
+ pWriter->iAbsLevel = iAbsLevel;
+ pWriter->iIdx = iIdx;
+
+ for(i=nHeight+1; i<FTS_MAX_APPENDABLE_HEIGHT; i++){
+ pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst;
+ }
+
+ pNode = &pWriter->aNodeWriter[nHeight];
+ pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight;
+ blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc);
+ if( rc==SQLITE_OK ){
+ memcpy(pNode->block.a, aRoot, nRoot);
+ pNode->block.n = nRoot;
+ }
+
+ for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){
+ NodeReader reader;
+ pNode = &pWriter->aNodeWriter[i];
+
+ rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n);
+ while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
+ blobGrowBuffer(&pNode->key, reader.term.n, &rc);
+ if( rc==SQLITE_OK ){
+ memcpy(pNode->key.a, reader.term.a, reader.term.n);
+ pNode->key.n = reader.term.n;
+ if( i>0 ){
+ char *aBlock = 0;
+ int nBlock = 0;
+ pNode = &pWriter->aNodeWriter[i-1];
+ pNode->iBlock = reader.iChild;
+ rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0);
+ blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc);
+ if( rc==SQLITE_OK ){
+ memcpy(pNode->block.a, aBlock, nBlock);
+ pNode->block.n = nBlock;
+ }
+ sqlite3_free(aBlock);
+ }
+ }
+ nodeReaderRelease(&reader);
+ }
+ }
+
+ rc2 = sqlite3_reset(pSelect);
+ if( rc==SQLITE_OK ) rc = rc2;
+ }
+
+ return rc;
+}
+
+/*
+** Determine the largest segment index value that exists within absolute
+** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus
+** one before returning SQLITE_OK. Or, if there are no segments at all
+** within level iAbsLevel, set *piIdx to zero.
+**
+** If an error occurs, return an SQLite error code. The final value of
+** *piIdx is undefined in this case.
+*/
+static int fts3IncrmergeOutputIdx(
+ Fts3Table *p, /* FTS Table handle */
+ sqlite3_int64 iAbsLevel, /* Absolute index of input segments */
+ int *piIdx /* OUT: Next free index at iAbsLevel+1 */
+){
+ int rc;
+ sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */
+
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1);
+ sqlite3_step(pOutputIdx);
+ *piIdx = sqlite3_column_int(pOutputIdx, 0);
+ rc = sqlite3_reset(pOutputIdx);
+ }
+
+ return rc;
+}
+
+/*
+** Allocate an appendable output segment on absolute level iAbsLevel+1
+** with idx value iIdx.
+**
+** In the %_segdir table, a segment is defined by the values in three
+** columns:
+**
+** start_block
+** leaves_end_block
+** end_block
+**
+** When an appendable segment is allocated, it is estimated that the
+** maximum number of leaf blocks that may be required is the sum of the
+** number of leaf blocks consumed by the input segments, plus the number
+** of input segments, multiplied by two. This value is stored in stack
+** variable nLeafEst.
+**
+** A total of 16*nLeafEst blocks are allocated when an appendable segment
+** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous
+** array of leaf nodes starts at the first block allocated. The array
+** of interior nodes that are parents of the leaf nodes start at block
+** (start_block + (1 + end_block - start_block) / 16). And so on.
+**
+** In the actual code below, the value "16" is replaced with the
+** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT.
+*/
+static int fts3IncrmergeWriter(
+ Fts3Table *p, /* Fts3 table handle */
+ sqlite3_int64 iAbsLevel, /* Absolute level of input segments */
+ int iIdx, /* Index of new output segment */
+ Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */
+ IncrmergeWriter *pWriter /* Populate this object */
+){
+ int rc; /* Return Code */
+ int i; /* Iterator variable */
+ int nLeafEst = 0; /* Blocks allocated for leaf nodes */
+ sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */
+ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */
+
+ /* Calculate nLeafEst. */
+ rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pLeafEst, 1, iAbsLevel);
+ sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment);
+ if( SQLITE_ROW==sqlite3_step(pLeafEst) ){
+ nLeafEst = sqlite3_column_int(pLeafEst, 0);
+ }
+ rc = sqlite3_reset(pLeafEst);
+ }
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* Calculate the first block to use in the output segment */
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){
+ pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0);
+ pWriter->iEnd = pWriter->iStart - 1;
+ pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT;
+ }
+ rc = sqlite3_reset(pFirstBlock);
+ }
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* Insert the marker in the %_segments table to make sure nobody tries
+ ** to steal the space just allocated. This is also used to identify
+ ** appendable segments. */
+ rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0);
+ if( rc!=SQLITE_OK ) return rc;
+
+ pWriter->iAbsLevel = iAbsLevel;
+ pWriter->nLeafEst = nLeafEst;
+ pWriter->iIdx = iIdx;
+
+ /* Set up the array of NodeWriter objects */
+ for(i=0; i<FTS_MAX_APPENDABLE_HEIGHT; i++){
+ pWriter->aNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Remove an entry from the %_segdir table. This involves running the
+** following two statements:
+**
+** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx
+** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx
+**
+** The DELETE statement removes the specific %_segdir level. The UPDATE
+** statement ensures that the remaining segments have contiguously allocated
+** idx values.
+*/
+static int fts3RemoveSegdirEntry(
+ Fts3Table *p, /* FTS3 table handle */
+ sqlite3_int64 iAbsLevel, /* Absolute level to delete from */
+ int iIdx /* Index of %_segdir entry to delete */
+){
+ int rc; /* Return code */
+ sqlite3_stmt *pDelete = 0; /* DELETE statement */
+
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pDelete, 1, iAbsLevel);
+ sqlite3_bind_int(pDelete, 2, iIdx);
+ sqlite3_step(pDelete);
+ rc = sqlite3_reset(pDelete);
+ }
+
+ return rc;
+}
+
+/*
+** One or more segments have just been removed from absolute level iAbsLevel.
+** Update the 'idx' values of the remaining segments in the level so that
+** the idx values are a contiguous sequence starting from 0.
+*/
+static int fts3RepackSegdirLevel(
+ Fts3Table *p, /* FTS3 table handle */
+ sqlite3_int64 iAbsLevel /* Absolute level to repack */
+){
+ int rc; /* Return code */
+ int *aIdx = 0; /* Array of remaining idx values */
+ int nIdx = 0; /* Valid entries in aIdx[] */
+ int nAlloc = 0; /* Allocated size of aIdx[] */
+ int i; /* Iterator variable */
+ sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */
+ sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */
+
+ rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0);
+ if( rc==SQLITE_OK ){
+ int rc2;
+ sqlite3_bind_int64(pSelect, 1, iAbsLevel);
+ while( SQLITE_ROW==sqlite3_step(pSelect) ){
+ if( nIdx>=nAlloc ){
+ int *aNew;
+ nAlloc += 16;
+ aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int));
+ if( !aNew ){
+ rc = SQLITE_NOMEM;
+ break;
+ }
+ aIdx = aNew;
+ }
+ aIdx[nIdx++] = sqlite3_column_int(pSelect, 0);
+ }
+ rc2 = sqlite3_reset(pSelect);
+ if( rc==SQLITE_OK ) rc = rc2;
+ }
+
+ if( rc==SQLITE_OK ){
+ rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0);
+ }
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pUpdate, 2, iAbsLevel);
+ }
+
+ assert( p->bIgnoreSavepoint==0 );
+ p->bIgnoreSavepoint = 1;
+ for(i=0; rc==SQLITE_OK && i<nIdx; i++){
+ if( aIdx[i]!=i ){
+ sqlite3_bind_int(pUpdate, 3, aIdx[i]);
+ sqlite3_bind_int(pUpdate, 1, i);
+ sqlite3_step(pUpdate);
+ rc = sqlite3_reset(pUpdate);
+ }
+ }
+ p->bIgnoreSavepoint = 0;
+
+ sqlite3_free(aIdx);
+ return rc;
+}
+
+static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){
+ pNode->a[0] = (char)iHeight;
+ if( iChild ){
+ assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) );
+ pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild);
+ }else{
+ assert( pNode->nAlloc>=1 );
+ pNode->n = 1;
+ }
+}
+
+/*
+** The first two arguments are a pointer to and the size of a segment b-tree
+** node. The node may be a leaf or an internal node.
+**
+** This function creates a new node image in blob object *pNew by copying
+** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes)
+** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode.
+*/
+static int fts3TruncateNode(
+ const char *aNode, /* Current node image */
+ int nNode, /* Size of aNode in bytes */
+ Blob *pNew, /* OUT: Write new node image here */
+ const char *zTerm, /* Omit all terms smaller than this */
+ int nTerm, /* Size of zTerm in bytes */
+ sqlite3_int64 *piBlock /* OUT: Block number in next layer down */
+){
+ NodeReader reader; /* Reader object */
+ Blob prev = {0, 0, 0}; /* Previous term written to new node */
+ int rc = SQLITE_OK; /* Return code */
+ int bLeaf = aNode[0]=='\0'; /* True for a leaf node */
+
+ /* Allocate required output space */
+ blobGrowBuffer(pNew, nNode, &rc);
+ if( rc!=SQLITE_OK ) return rc;
+ pNew->n = 0;
+
+ /* Populate new node buffer */
+ for(rc = nodeReaderInit(&reader, aNode, nNode);
+ rc==SQLITE_OK && reader.aNode;
+ rc = nodeReaderNext(&reader)
+ ){
+ if( pNew->n==0 ){
+ int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm);
+ if( res<0 || (bLeaf==0 && res==0) ) continue;
+ fts3StartNode(pNew, (int)aNode[0], reader.iChild);
+ *piBlock = reader.iChild;
+ }
+ rc = fts3AppendToNode(
+ pNew, &prev, reader.term.a, reader.term.n,
+ reader.aDoclist, reader.nDoclist
+ );
+ if( rc!=SQLITE_OK ) break;
+ }
+ if( pNew->n==0 ){
+ fts3StartNode(pNew, (int)aNode[0], reader.iChild);
+ *piBlock = reader.iChild;
+ }
+ assert( pNew->n<=pNew->nAlloc );
+
+ nodeReaderRelease(&reader);
+ sqlite3_free(prev.a);
+ return rc;
+}
+
+/*
+** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute
+** level iAbsLevel. This may involve deleting entries from the %_segments
+** table, and modifying existing entries in both the %_segments and %_segdir
+** tables.
+**
+** SQLITE_OK is returned if the segment is updated successfully. Or an
+** SQLite error code otherwise.
+*/
+static int fts3TruncateSegment(
+ Fts3Table *p, /* FTS3 table handle */
+ sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */
+ int iIdx, /* Index within level of segment to modify */
+ const char *zTerm, /* Remove terms smaller than this */
+ int nTerm /* Number of bytes in buffer zTerm */
+){
+ int rc = SQLITE_OK; /* Return code */
+ Blob root = {0,0,0}; /* New root page image */
+ Blob block = {0,0,0}; /* Buffer used for any other block */
+ sqlite3_int64 iBlock = 0; /* Block id */
+ sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */
+ sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */
+ sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */
+
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0);
+ if( rc==SQLITE_OK ){
+ int rc2; /* sqlite3_reset() return code */
+ sqlite3_bind_int64(pFetch, 1, iAbsLevel);
+ sqlite3_bind_int(pFetch, 2, iIdx);
+ if( SQLITE_ROW==sqlite3_step(pFetch) ){
+ const char *aRoot = sqlite3_column_blob(pFetch, 4);
+ int nRoot = sqlite3_column_bytes(pFetch, 4);
+ iOldStart = sqlite3_column_int64(pFetch, 1);
+ rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock);
+ }
+ rc2 = sqlite3_reset(pFetch);
+ if( rc==SQLITE_OK ) rc = rc2;
+ }
+
+ while( rc==SQLITE_OK && iBlock ){
+ char *aBlock = 0;
+ int nBlock = 0;
+ iNewStart = iBlock;
+
+ rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0);
+ if( rc==SQLITE_OK ){
+ rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3WriteSegment(p, iNewStart, block.a, block.n);
+ }
+ sqlite3_free(aBlock);
+ }
+
+ /* Variable iNewStart now contains the first valid leaf node. */
+ if( rc==SQLITE_OK && iNewStart ){
+ sqlite3_stmt *pDel = 0;
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pDel, 1, iOldStart);
+ sqlite3_bind_int64(pDel, 2, iNewStart-1);
+ sqlite3_step(pDel);
+ rc = sqlite3_reset(pDel);
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ sqlite3_stmt *pChomp = 0;
+ rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pChomp, 1, iNewStart);
+ sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC);
+ sqlite3_bind_int64(pChomp, 3, iAbsLevel);
+ sqlite3_bind_int(pChomp, 4, iIdx);
+ sqlite3_step(pChomp);
+ rc = sqlite3_reset(pChomp);
+ }
+ }
+
+ sqlite3_free(root.a);
+ sqlite3_free(block.a);
+ return rc;
+}
+
+/*
+** This function is called after an incrmental-merge operation has run to
+** merge (or partially merge) two or more segments from absolute level
+** iAbsLevel.
+**
+** Each input segment is either removed from the db completely (if all of
+** its data was copied to the output segment by the incrmerge operation)
+** or modified in place so that it no longer contains those entries that
+** have been duplicated in the output segment.
+*/
+static int fts3IncrmergeChomp(
+ Fts3Table *p, /* FTS table handle */
+ sqlite3_int64 iAbsLevel, /* Absolute level containing segments */
+ Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */
+ int *pnRem /* Number of segments not deleted */
+){
+ int i;
+ int nRem = 0;
+ int rc = SQLITE_OK;
+
+ for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){
+ Fts3SegReader *pSeg = 0;
+ int j;
+
+ /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding
+ ** somewhere in the pCsr->apSegment[] array. */
+ for(j=0; ALWAYS(j<pCsr->nSegment); j++){
+ pSeg = pCsr->apSegment[j];
+ if( pSeg->iIdx==i ) break;
+ }
+ assert( j<pCsr->nSegment && pSeg->iIdx==i );
+
+ if( pSeg->aNode==0 ){
+ /* Seg-reader is at EOF. Remove the entire input segment. */
+ rc = fts3DeleteSegment(p, pSeg);
+ if( rc==SQLITE_OK ){
+ rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx);
+ }
+ *pnRem = 0;
+ }else{
+ /* The incremental merge did not copy all the data from this
+ ** segment to the upper level. The segment is modified in place
+ ** so that it contains no keys smaller than zTerm/nTerm. */
+ const char *zTerm = pSeg->zTerm;
+ int nTerm = pSeg->nTerm;
+ rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm);
+ nRem++;
+ }
+ }
+
+ if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){
+ rc = fts3RepackSegdirLevel(p, iAbsLevel);
+ }
+
+ *pnRem = nRem;
+ return rc;
+}
+
+/*
+** Store an incr-merge hint in the database.
+*/
+static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){
+ sqlite3_stmt *pReplace = 0;
+ int rc; /* Return code */
+
+ rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT);
+ sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
+ sqlite3_step(pReplace);
+ rc = sqlite3_reset(pReplace);
+ }
+
+ return rc;
+}
+
+/*
+** Load an incr-merge hint from the database. The incr-merge hint, if one
+** exists, is stored in the rowid==1 row of the %_stat table.
+**
+** If successful, populate blob *pHint with the value read from the %_stat
+** table and return SQLITE_OK. Otherwise, if an error occurs, return an
+** SQLite error code.
+*/
+static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){
+ sqlite3_stmt *pSelect = 0;
+ int rc;
+
+ pHint->n = 0;
+ rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0);
+ if( rc==SQLITE_OK ){
+ int rc2;
+ sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT);
+ if( SQLITE_ROW==sqlite3_step(pSelect) ){
+ const char *aHint = sqlite3_column_blob(pSelect, 0);
+ int nHint = sqlite3_column_bytes(pSelect, 0);
+ if( aHint ){
+ blobGrowBuffer(pHint, nHint, &rc);
+ if( rc==SQLITE_OK ){
+ memcpy(pHint->a, aHint, nHint);
+ pHint->n = nHint;
+ }
+ }
+ }
+ rc2 = sqlite3_reset(pSelect);
+ if( rc==SQLITE_OK ) rc = rc2;
+ }
+
+ return rc;
+}
+
+/*
+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
+** Otherwise, append an entry to the hint stored in blob *pHint. Each entry
+** consists of two varints, the absolute level number of the input segments
+** and the number of input segments.
+**
+** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs,
+** set *pRc to an SQLite error code before returning.
+*/
+static void fts3IncrmergeHintPush(
+ Blob *pHint, /* Hint blob to append to */
+ i64 iAbsLevel, /* First varint to store in hint */
+ int nInput, /* Second varint to store in hint */
+ int *pRc /* IN/OUT: Error code */
+){
+ blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc);
+ if( *pRc==SQLITE_OK ){
+ pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel);
+ pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput);
+ }
+}
+
+/*
+** Read the last entry (most recently pushed) from the hint blob *pHint
+** and then remove the entry. Write the two values read to *piAbsLevel and
+** *pnInput before returning.
+**
+** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does
+** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB.
+*/
+static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){
+ const int nHint = pHint->n;
+ int i;
+
+ i = pHint->n-2;
+ while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
+ while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
+
+ pHint->n = i;
+ i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
+ i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput);
+ if( i!=nHint ) return SQLITE_CORRUPT_VTAB;
+
+ return SQLITE_OK;
+}
+
+
+/*
+** Attempt an incremental merge that writes nMerge leaf blocks.
+**
+** Incremental merges happen nMin segments at a time. The two
+** segments to be merged are the nMin oldest segments (the ones with
+** the smallest indexes) in the highest level that contains at least
+** nMin segments. Multiple merges might occur in an attempt to write the
+** quota of nMerge leaf blocks.
+*/
+int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
+ int rc; /* Return code */
+ int nRem = nMerge; /* Number of leaf pages yet to be written */
+ Fts3MultiSegReader *pCsr; /* Cursor used to read input data */
+ Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */
+ IncrmergeWriter *pWriter; /* Writer object */
+ int nSeg = 0; /* Number of input segments */
+ sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */
+ Blob hint = {0, 0, 0}; /* Hint read from %_stat table */
+ int bDirtyHint = 0; /* True if blob 'hint' has been modified */
+
+ /* Allocate space for the cursor, filter and writer objects */
+ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter);
+ pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc);
+ if( !pWriter ) return SQLITE_NOMEM;
+ pFilter = (Fts3SegFilter *)&pWriter[1];
+ pCsr = (Fts3MultiSegReader *)&pFilter[1];
+
+ rc = fts3IncrmergeHintLoad(p, &hint);
+ while( rc==SQLITE_OK && nRem>0 ){
+ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex;
+ sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */
+ int bUseHint = 0; /* True if attempting to append */
+
+ /* Search the %_segdir table for the absolute level with the smallest
+ ** relative level number that contains at least nMin segments, if any.
+ ** If one is found, set iAbsLevel to the absolute level number and
+ ** nSeg to nMin. If no level with at least nMin segments can be found,
+ ** set nSeg to -1.
+ */
+ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0);
+ sqlite3_bind_int(pFindLevel, 1, nMin);
+ if( sqlite3_step(pFindLevel)==SQLITE_ROW ){
+ iAbsLevel = sqlite3_column_int64(pFindLevel, 0);
+ nSeg = nMin;
+ }else{
+ nSeg = -1;
+ }
+ rc = sqlite3_reset(pFindLevel);
+
+ /* If the hint read from the %_stat table is not empty, check if the
+ ** last entry in it specifies a relative level smaller than or equal
+ ** to the level identified by the block above (if any). If so, this
+ ** iteration of the loop will work on merging at the hinted level.
+ */
+ if( rc==SQLITE_OK && hint.n ){
+ int nHint = hint.n;
+ sqlite3_int64 iHintAbsLevel = 0; /* Hint level */
+ int nHintSeg = 0; /* Hint number of segments */
+
+ rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg);
+ if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){
+ iAbsLevel = iHintAbsLevel;
+ nSeg = nHintSeg;
+ bUseHint = 1;
+ bDirtyHint = 1;
+ }else{
+ /* This undoes the effect of the HintPop() above - so that no entry
+ ** is removed from the hint blob. */
+ hint.n = nHint;
+ }
+ }
+
+ /* If nSeg is less that zero, then there is no level with at least
+ ** nMin segments and no hint in the %_stat table. No work to do.
+ ** Exit early in this case. */
+ if( nSeg<0 ) break;
+
+ /* Open a cursor to iterate through the contents of the oldest nSeg
+ ** indexes of absolute level iAbsLevel. If this cursor is opened using
+ ** the 'hint' parameters, it is possible that there are less than nSeg
+ ** segments available in level iAbsLevel. In this case, no work is
+ ** done on iAbsLevel - fall through to the next iteration of the loop
+ ** to start work on some other level. */
+ memset(pWriter, 0, nAlloc);
+ pFilter->flags = FTS3_SEGMENT_REQUIRE_POS;
+ if( rc==SQLITE_OK ){
+ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);
+ }
+ if( SQLITE_OK==rc && pCsr->nSegment==nSeg
+ && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
+ && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
+ ){
+ int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */
+ rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
+ if( rc==SQLITE_OK ){
+ if( bUseHint && iIdx>0 ){
+ const char *zKey = pCsr->zTerm;
+ int nKey = pCsr->nTerm;
+ rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter);
+ }else{
+ rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter);
+ }
+ }
+
+ if( rc==SQLITE_OK && pWriter->nLeafEst ){
+ fts3LogMerge(nSeg, iAbsLevel);
+ do {
+ rc = fts3IncrmergeAppend(p, pWriter, pCsr);
+ if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
+ if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
+ }while( rc==SQLITE_ROW );
+
+ /* Update or delete the input segments */
+ if( rc==SQLITE_OK ){
+ nRem -= (1 + pWriter->nWork);
+ rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg);
+ if( nSeg!=0 ){
+ bDirtyHint = 1;
+ fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc);
+ }
+ }
+ }
+
+ fts3IncrmergeRelease(p, pWriter, &rc);
+ }
+
+ sqlite3Fts3SegReaderFinish(pCsr);
+ }
+
+ /* Write the hint values into the %_stat table for the next incr-merger */
+ if( bDirtyHint && rc==SQLITE_OK ){
+ rc = fts3IncrmergeHintStore(p, &hint);
+ }
+
+ sqlite3_free(pWriter);
+ sqlite3_free(hint.a);
+ return rc;
+}
+
+/*
+** Convert the text beginning at *pz into an integer and return
+** its value. Advance *pz to point to the first character past
+** the integer.
+*/
+static int fts3Getint(const char **pz){
+ const char *z = *pz;
+ int i = 0;
+ while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0';
+ *pz = z;
+ return i;
+}
+
+/*
+** Process statements of the form:
+**
+** INSERT INTO table(table) VALUES('merge=A,B');
+**
+** A and B are integers that decode to be the number of leaf pages
+** written for the merge, and the minimum number of segments on a level
+** before it will be selected for a merge, respectively.
+*/
+static int fts3DoIncrmerge(
+ Fts3Table *p, /* FTS3 table handle */
+ const char *zParam /* Nul-terminated string containing "A,B" */
+){
+ int rc;
+ int nMin = (FTS3_MERGE_COUNT / 2);
+ int nMerge = 0;
+ const char *z = zParam;
+
+ /* Read the first integer value */
+ nMerge = fts3Getint(&z);
+
+ /* If the first integer value is followed by a ',', read the second
+ ** integer value. */
+ if( z[0]==',' && z[1]!='\0' ){
+ z++;
+ nMin = fts3Getint(&z);
+ }
+
+ if( z[0]!='\0' || nMin<2 ){
+ rc = SQLITE_ERROR;
+ }else{
+ rc = SQLITE_OK;
+ if( !p->bHasStat ){
+ assert( p->bFts4==0 );
+ sqlite3Fts3CreateStatTable(&rc, p);
+ }
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3Incrmerge(p, nMerge, nMin);
+ }
+ sqlite3Fts3SegmentsClose(p);
+ }
+ return rc;
+}
+
+/*
+** Process statements of the form:
+**
+** INSERT INTO table(table) VALUES('automerge=X');
+**
+** where X is an integer. X==0 means to turn automerge off. X!=0 means
+** turn it on. The setting is persistent.
+*/
+static int fts3DoAutoincrmerge(
+ Fts3Table *p, /* FTS3 table handle */
+ const char *zParam /* Nul-terminated string containing boolean */
+){
+ int rc = SQLITE_OK;
+ sqlite3_stmt *pStmt = 0;
+ p->bAutoincrmerge = fts3Getint(&zParam)!=0;
+ if( !p->bHasStat ){
+ assert( p->bFts4==0 );
+ sqlite3Fts3CreateStatTable(&rc, p);
+ if( rc ) return rc;
+ }
+ rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
+ if( rc ) return rc;;
+ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
+ sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge);
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ return rc;
+}
+
+/*
+** Return a 64-bit checksum for the FTS index entry specified by the
+** arguments to this function.
+*/
+static u64 fts3ChecksumEntry(
+ const char *zTerm, /* Pointer to buffer containing term */
+ int nTerm, /* Size of zTerm in bytes */
+ int iLangid, /* Language id for current row */
+ int iIndex, /* Index (0..Fts3Table.nIndex-1) */
+ i64 iDocid, /* Docid for current row. */
+ int iCol, /* Column number */
+ int iPos /* Position */
+){
+ int i;
+ u64 ret = (u64)iDocid;
+
+ ret += (ret<<3) + iLangid;
+ ret += (ret<<3) + iIndex;
+ ret += (ret<<3) + iCol;
+ ret += (ret<<3) + iPos;
+ for(i=0; i<nTerm; i++) ret += (ret<<3) + zTerm[i];
+
+ return ret;
+}
+
+/*
+** Return a checksum of all entries in the FTS index that correspond to
+** language id iLangid. The checksum is calculated by XORing the checksums
+** of each individual entry (see fts3ChecksumEntry()) together.
+**
+** If successful, the checksum value is returned and *pRc set to SQLITE_OK.
+** Otherwise, if an error occurs, *pRc is set to an SQLite error code. The
+** return value is undefined in this case.
+*/
+static u64 fts3ChecksumIndex(
+ Fts3Table *p, /* FTS3 table handle */
+ int iLangid, /* Language id to return cksum for */
+ int iIndex, /* Index to cksum (0..p->nIndex-1) */
+ int *pRc /* OUT: Return code */
+){
+ Fts3SegFilter filter;
+ Fts3MultiSegReader csr;
+ int rc;
+ u64 cksum = 0;
+
+ assert( *pRc==SQLITE_OK );
+
+ memset(&filter, 0, sizeof(filter));
+ memset(&csr, 0, sizeof(csr));
+ filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
+ filter.flags |= FTS3_SEGMENT_SCAN;
+
+ rc = sqlite3Fts3SegReaderCursor(
+ p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr
+ );
+ if( rc==SQLITE_OK ){
+ rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
+ }
+
+ if( rc==SQLITE_OK ){
+ while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){
+ char *pCsr = csr.aDoclist;
+ char *pEnd = &pCsr[csr.nDoclist];
+
+ i64 iDocid = 0;
+ i64 iCol = 0;
+ i64 iPos = 0;
+
+ pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid);
+ while( pCsr<pEnd ){
+ i64 iVal = 0;
+ pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
+ if( pCsr<pEnd ){
+ if( iVal==0 || iVal==1 ){
+ iCol = 0;
+ iPos = 0;
+ if( iVal ){
+ pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
+ }else{
+ pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
+ iDocid += iVal;
+ }
+ }else{
+ iPos += (iVal - 2);
+ cksum = cksum ^ fts3ChecksumEntry(
+ csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid,
+ (int)iCol, (int)iPos
+ );
+ }
+ }
+ }
+ }
+ }
+ sqlite3Fts3SegReaderFinish(&csr);
+
+ *pRc = rc;
+ return cksum;
+}
+
+/*
+** Check if the contents of the FTS index match the current contents of the
+** content table. If no error occurs and the contents do match, set *pbOk
+** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk
+** to false before returning.
+**
+** If an error occurs (e.g. an OOM or IO error), return an SQLite error
+** code. The final value of *pbOk is undefined in this case.
+*/
+static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
+ int rc = SQLITE_OK; /* Return code */
+ u64 cksum1 = 0; /* Checksum based on FTS index contents */
+ u64 cksum2 = 0; /* Checksum based on %_content contents */
+ sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */
+
+ /* This block calculates the checksum according to the FTS index. */
+ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
+ if( rc==SQLITE_OK ){
+ int rc2;
+ sqlite3_bind_int(pAllLangid, 1, p->nIndex);
+ while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){
+ int iLangid = sqlite3_column_int(pAllLangid, 0);
+ int i;
+ for(i=0; i<p->nIndex; i++){
+ cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc);
+ }
+ }
+ rc2 = sqlite3_reset(pAllLangid);
+ if( rc==SQLITE_OK ) rc = rc2;
+ }
+
+ /* This block calculates the checksum according to the %_content table */
+ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule;
+ sqlite3_stmt *pStmt = 0;
+ char *zSql;
+
+ zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+ sqlite3_free(zSql);
+ }
+
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
+ i64 iDocid = sqlite3_column_int64(pStmt, 0);
+ int iLang = langidFromSelect(p, pStmt);
+ int iCol;
+
+ for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
+ const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1);
+ int nText = sqlite3_column_bytes(pStmt, iCol+1);
+ sqlite3_tokenizer_cursor *pT = 0;
+
+ rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText, &pT);
+ while( rc==SQLITE_OK ){
+ char const *zToken; /* Buffer containing token */
+ int nToken; /* Number of bytes in token */
+ int iDum1, iDum2; /* Dummy variables */
+ int iPos; /* Position of token in zText */
+
+ rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos);
+ if( rc==SQLITE_OK ){
+ int i;
+ cksum2 = cksum2 ^ fts3ChecksumEntry(
+ zToken, nToken, iLang, 0, iDocid, iCol, iPos
+ );
+ for(i=1; i<p->nIndex; i++){
+ if( p->aIndex[i].nPrefix<=nToken ){
+ cksum2 = cksum2 ^ fts3ChecksumEntry(
+ zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos
+ );
+ }
+ }
+ }
+ }
+ if( pT ) pModule->xClose(pT);
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+ }
+ }
+
+ sqlite3_finalize(pStmt);
+ }
+
+ *pbOk = (cksum1==cksum2);
+ return rc;
+}
+
+/*
+** Run the integrity-check. If no error occurs and the current contents of
+** the FTS index are correct, return SQLITE_OK. Or, if the contents of the
+** FTS index are incorrect, return SQLITE_CORRUPT_VTAB.
+**
+** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite
+** error code.
+**
+** The integrity-check works as follows. For each token and indexed token
+** prefix in the document set, a 64-bit checksum is calculated (by code
+** in fts3ChecksumEntry()) based on the following:
+**
+** + The index number (0 for the main index, 1 for the first prefix
+** index etc.),
+** + The token (or token prefix) text itself,
+** + The language-id of the row it appears in,
+** + The docid of the row it appears in,
+** + The column it appears in, and
+** + The tokens position within that column.
+**
+** The checksums for all entries in the index are XORed together to create
+** a single checksum for the entire index.
+**
+** The integrity-check code calculates the same checksum in two ways:
+**
+** 1. By scanning the contents of the FTS index, and
+** 2. By scanning and tokenizing the content table.
+**
+** If the two checksums are identical, the integrity-check is deemed to have
+** passed.
+*/
+static int fts3DoIntegrityCheck(
+ Fts3Table *p /* FTS3 table handle */
+){
+ int rc;
+ int bOk = 0;
+ rc = fts3IntegrityCheck(p, &bOk);
+ if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_CORRUPT_VTAB;
+ return rc;
+}
+
/*
** Handle a 'special' INSERT of the form:
**
@@ -3051,6 +5018,12 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
rc = fts3DoOptimize(p, 0);
}else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){
rc = fts3DoRebuild(p);
+ }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){
+ rc = fts3DoIntegrityCheck(p);
+ }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){
+ rc = fts3DoIncrmerge(p, &zVal[6]);
+ }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
+ rc = fts3DoAutoincrmerge(p, &zVal[10]);
#ifdef SQLITE_TEST
}else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
p->nNodeSize = atoi(&zVal[9]);
@@ -3119,14 +5092,13 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
sqlite3_tokenizer_cursor *pTC = 0;
- rc = pModule->xOpen(pT, zText, -1, &pTC);
+ rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC);
while( rc==SQLITE_OK ){
char const *zToken; /* Buffer containing token */
int nToken; /* Number of bytes in token */
int iDum1, iDum2; /* Dummy variables */
int iPos; /* Position of token in zText */
- pTC->pTokenizer = pT;
rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos);
for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
Fts3PhraseToken *pPT = pDef->pToken;
@@ -3226,8 +5198,6 @@ static int fts3DeleteByRowid(
rc = fts3DeleteAll(p, 1);
*pnDoc = *pnDoc - 1;
}else{
- sqlite3_int64 iRemove = sqlite3_value_int64(pRowid);
- rc = fts3PendingTermsDocid(p, iRemove);
fts3DeleteTerms(&rc, p, pRowid, aSzDel);
if( p->zContentTbl==0 ){
fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
@@ -3246,7 +5216,16 @@ static int fts3DeleteByRowid(
/*
** This function does the work for the xUpdate method of FTS3 virtual
-** tables.
+** tables. The schema of the virtual table being:
+**
+** CREATE TABLE <table name>(
+** <user columns>,
+** <table name> HIDDEN,
+** docid HIDDEN,
+** <langid> HIDDEN
+** );
+**
+**
*/
int sqlite3Fts3UpdateMethod(
sqlite3_vtab *pVtab, /* FTS3 vtab object */
@@ -3263,6 +5242,10 @@ int sqlite3Fts3UpdateMethod(
int bInsertDone = 0;
assert( p->pSegments==0 );
+ assert(
+ nArg==1 /* DELETE operations */
+ || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */
+ );
/* Check for a "special" INSERT operation. One of the form:
**
@@ -3276,6 +5259,11 @@ int sqlite3Fts3UpdateMethod(
goto update_out;
}
+ if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){
+ rc = SQLITE_CONSTRAINT;
+ goto update_out;
+ }
+
/* Allocate space to hold the change in document sizes */
aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
if( aSzIns==0 ){
@@ -3343,6 +5331,7 @@ int sqlite3Fts3UpdateMethod(
/* If this is an INSERT or UPDATE operation, insert the new record. */
if( nArg>1 && rc==SQLITE_OK ){
+ int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]);
if( bInsertDone==0 ){
rc = fts3InsertData(p, apVal, pRowid);
if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
@@ -3350,11 +5339,11 @@ int sqlite3Fts3UpdateMethod(
}
}
if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
- rc = fts3PendingTermsDocid(p, *pRowid);
+ rc = fts3PendingTermsDocid(p, iLangid, *pRowid);
}
if( rc==SQLITE_OK ){
assert( p->iPrevDocid==*pRowid );
- rc = fts3InsertTerms(p, apVal, aSzIns);
+ rc = fts3InsertTerms(p, iLangid, apVal, aSzIns);
}
if( p->bHasDocsize ){
fts3InsertDocsize(&rc, p, aSzIns);
@@ -3362,7 +5351,7 @@ int sqlite3Fts3UpdateMethod(
nChng++;
}
- if( p->bHasStat ){
+ if( p->bFts4 ){
fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
}
diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c
new file mode 100644
index 0000000..479ae98
--- /dev/null
+++ b/ext/fts3/tool/fts3view.c
@@ -0,0 +1,874 @@
+/*
+** This program is a debugging and analysis utility that displays
+** information about an FTS3 or FTS4 index.
+**
+** Link this program against the SQLite3 amalgamation with the
+** SQLITE_ENABLE_FTS4 compile-time option. Then run it as:
+**
+** fts3view DATABASE
+**
+** to get a list of all FTS3/4 tables in DATABASE, or do
+**
+** fts3view DATABASE TABLE COMMAND ....
+**
+** to see various aspects of the TABLE table. Type fts3view with no
+** arguments for a list of available COMMANDs.
+*/
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include "sqlite3.h"
+
+/*
+** Extra command-line arguments:
+*/
+int nExtra;
+char **azExtra;
+
+/*
+** Look for a command-line argument.
+*/
+const char *findOption(const char *zName, int hasArg, const char *zDefault){
+ int i;
+ const char *zResult = zDefault;
+ for(i=0; i<nExtra; i++){
+ const char *z = azExtra[i];
+ while( z[0]=='-' ) z++;
+ if( strcmp(z, zName)==0 ){
+ int j = 1;
+ if( hasArg==0 || i==nExtra-1 ) j = 0;
+ zResult = azExtra[i+j];
+ while( i+j<nExtra ){
+ azExtra[i] = azExtra[i+j+1];
+ i++;
+ }
+ break;
+ }
+ }
+ return zResult;
+}
+
+
+/*
+** Prepare an SQL query
+*/
+static sqlite3_stmt *prepare(sqlite3 *db, const char *zFormat, ...){
+ va_list ap;
+ char *zSql;
+ sqlite3_stmt *pStmt;
+ int rc;
+
+ va_start(ap, zFormat);
+ zSql = sqlite3_vmprintf(zFormat, ap);
+ va_end(ap);
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ if( rc ){
+ fprintf(stderr, "Error: %s\nSQL: %s\n", sqlite3_errmsg(db), zSql);
+ exit(1);
+ }
+ sqlite3_free(zSql);
+ return pStmt;
+}
+
+/*
+** Run an SQL statement
+*/
+static int runSql(sqlite3 *db, const char *zFormat, ...){
+ va_list ap;
+ char *zSql;
+ int rc;
+
+ va_start(ap, zFormat);
+ zSql = sqlite3_vmprintf(zFormat, ap);
+ rc = sqlite3_exec(db, zSql, 0, 0, 0);
+ va_end(ap);
+ return rc;
+}
+
+/*
+** Show the table schema
+*/
+static void showSchema(sqlite3 *db, const char *zTab){
+ sqlite3_stmt *pStmt;
+ pStmt = prepare(db,
+ "SELECT sql FROM sqlite_master"
+ " WHERE name LIKE '%q%%'"
+ " ORDER BY 1",
+ zTab);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ printf("%s;\n", sqlite3_column_text(pStmt, 0));
+ }
+ sqlite3_finalize(pStmt);
+ pStmt = prepare(db, "PRAGMA page_size");
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ printf("PRAGMA page_size=%s;\n", sqlite3_column_text(pStmt, 0));
+ }
+ sqlite3_finalize(pStmt);
+ pStmt = prepare(db, "PRAGMA journal_mode");
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ printf("PRAGMA journal_mode=%s;\n", sqlite3_column_text(pStmt, 0));
+ }
+ sqlite3_finalize(pStmt);
+ pStmt = prepare(db, "PRAGMA auto_vacuum");
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ const char *zType = "???";
+ switch( sqlite3_column_int(pStmt, 0) ){
+ case 0: zType = "OFF"; break;
+ case 1: zType = "FULL"; break;
+ case 2: zType = "INCREMENTAL"; break;
+ }
+ printf("PRAGMA auto_vacuum=%s;\n", zType);
+ }
+ sqlite3_finalize(pStmt);
+ pStmt = prepare(db, "PRAGMA encoding");
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ printf("PRAGMA encoding=%s;\n", sqlite3_column_text(pStmt, 0));
+ }
+ sqlite3_finalize(pStmt);
+}
+
+/*
+** Read a 64-bit variable-length integer from memory starting at p[0].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+int getVarint(const unsigned char *p, sqlite_int64 *v){
+ const unsigned char *q = p;
+ sqlite_uint64 x = 0, y = 1;
+ while( (*q&0x80)==0x80 && q-(unsigned char *)p<9 ){
+ x += y * (*q++ & 0x7f);
+ y <<= 7;
+ }
+ x += y * (*q++);
+ *v = (sqlite_int64) x;
+ return (int) (q - (unsigned char *)p);
+}
+
+
+/* Show the content of the %_stat table
+*/
+static void showStat(sqlite3 *db, const char *zTab){
+ sqlite3_stmt *pStmt;
+ pStmt = prepare(db, "SELECT id, value FROM '%q_stat'", zTab);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ printf("stat[%d] =", sqlite3_column_int(pStmt, 0));
+ switch( sqlite3_column_type(pStmt, 1) ){
+ case SQLITE_INTEGER: {
+ printf(" %d\n", sqlite3_column_int(pStmt, 1));
+ break;
+ }
+ case SQLITE_BLOB: {
+ unsigned char *x = (unsigned char*)sqlite3_column_blob(pStmt, 1);
+ int len = sqlite3_column_bytes(pStmt, 1);
+ int i = 0;
+ sqlite3_int64 v;
+ while( i<len ){
+ i += getVarint(x, &v);
+ printf(" %lld", v);
+ }
+ printf("\n");
+ break;
+ }
+ }
+ }
+ sqlite3_finalize(pStmt);
+}
+
+/*
+** Report on the vocabulary. This creates an fts4aux table with a random
+** name, but deletes it in the end.
+*/
+static void showVocabulary(sqlite3 *db, const char *zTab){
+ char *zAux;
+ sqlite3_uint64 r;
+ sqlite3_stmt *pStmt;
+ int nDoc = 0;
+ int nToken = 0;
+ int nOccurrence = 0;
+ int nTop;
+ int n, i;
+
+ sqlite3_randomness(sizeof(r), &r);
+ zAux = sqlite3_mprintf("viewer_%llx", zTab, r);
+ runSql(db, "BEGIN");
+ pStmt = prepare(db, "SELECT count(*) FROM %Q", zTab);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ nDoc = sqlite3_column_int(pStmt, 0);
+ }
+ sqlite3_finalize(pStmt);
+ printf("Number of documents...................... %9d\n", nDoc);
+
+ runSql(db, "CREATE VIRTUAL TABLE %s USING fts4aux(%Q)", zAux, zTab);
+ pStmt = prepare(db,
+ "SELECT count(*), sum(occurrences) FROM %s WHERE col='*'",
+ zAux);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ nToken = sqlite3_column_int(pStmt, 0);
+ nOccurrence = sqlite3_column_int(pStmt, 1);
+ }
+ sqlite3_finalize(pStmt);
+ printf("Total tokens in all documents............ %9d\n", nOccurrence);
+ printf("Total number of distinct tokens.......... %9d\n", nToken);
+ if( nToken==0 ) goto end_vocab;
+
+ n = 0;
+ pStmt = prepare(db, "SELECT count(*) FROM %s"
+ " WHERE col='*' AND occurrences==1", zAux);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ n = sqlite3_column_int(pStmt, 0);
+ }
+ sqlite3_finalize(pStmt);
+ printf("Tokens used exactly once................. %9d %5.2f%%\n",
+ n, n*100.0/nToken);
+
+ n = 0;
+ pStmt = prepare(db, "SELECT count(*) FROM %s"
+ " WHERE col='*' AND documents==1", zAux);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ n = sqlite3_column_int(pStmt, 0);
+ }
+ sqlite3_finalize(pStmt);
+ printf("Tokens used in only one document......... %9d %5.2f%%\n",
+ n, n*100.0/nToken);
+
+ if( nDoc>=2000 ){
+ n = 0;
+ pStmt = prepare(db, "SELECT count(*) FROM %s"
+ " WHERE col='*' AND occurrences<=%d", zAux, nDoc/1000);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ n = sqlite3_column_int(pStmt, 0);
+ }
+ sqlite3_finalize(pStmt);
+ printf("Tokens used in 0.1%% or less of docs...... %9d %5.2f%%\n",
+ n, n*100.0/nToken);
+ }
+
+ if( nDoc>=200 ){
+ n = 0;
+ pStmt = prepare(db, "SELECT count(*) FROM %s"
+ " WHERE col='*' AND occurrences<=%d", zAux, nDoc/100);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ n = sqlite3_column_int(pStmt, 0);
+ }
+ sqlite3_finalize(pStmt);
+ printf("Tokens used in 1%% or less of docs........ %9d %5.2f%%\n",
+ n, n*100.0/nToken);
+ }
+
+ nTop = atoi(findOption("top", 1, "25"));
+ printf("The %d most common tokens:\n", nTop);
+ pStmt = prepare(db,
+ "SELECT term, documents FROM %s"
+ " WHERE col='*'"
+ " ORDER BY documents DESC, term"
+ " LIMIT %d", zAux, nTop);
+ i = 0;
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ i++;
+ n = sqlite3_column_int(pStmt, 1);
+ printf(" %2d. %-30s %9d docs %5.2f%%\n", i,
+ sqlite3_column_text(pStmt, 0), n, n*100.0/nDoc);
+ }
+ sqlite3_finalize(pStmt);
+
+end_vocab:
+ runSql(db, "ROLLBACK");
+ sqlite3_free(zAux);
+}
+
+/*
+** Report on the number and sizes of segments
+*/
+static void showSegmentStats(sqlite3 *db, const char *zTab){
+ sqlite3_stmt *pStmt;
+ int nSeg = 0;
+ sqlite3_int64 szSeg = 0, mxSeg = 0;
+ int nIdx = 0;
+ sqlite3_int64 szIdx = 0, mxIdx = 0;
+ int nRoot = 0;
+ sqlite3_int64 szRoot = 0, mxRoot = 0;
+ sqlite3_int64 mx;
+ int nLeaf;
+ int n;
+ int pgsz;
+ int mxLevel;
+ int i;
+
+ pStmt = prepare(db,
+ "SELECT count(*), sum(length(block)), max(length(block))"
+ " FROM '%q_segments'",
+ zTab);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ nSeg = sqlite3_column_int(pStmt, 0);
+ szSeg = sqlite3_column_int64(pStmt, 1);
+ mxSeg = sqlite3_column_int64(pStmt, 2);
+ }
+ sqlite3_finalize(pStmt);
+ pStmt = prepare(db,
+ "SELECT count(*), sum(length(block)), max(length(block))"
+ " FROM '%q_segments' a JOIN '%q_segdir' b"
+ " WHERE a.blockid BETWEEN b.leaves_end_block+1 AND b.end_block",
+ zTab, zTab);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ nIdx = sqlite3_column_int(pStmt, 0);
+ szIdx = sqlite3_column_int64(pStmt, 1);
+ mxIdx = sqlite3_column_int64(pStmt, 2);
+ }
+ sqlite3_finalize(pStmt);
+ pStmt = prepare(db,
+ "SELECT count(*), sum(length(root)), max(length(root))"
+ " FROM '%q_segdir'",
+ zTab);
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ nRoot = sqlite3_column_int(pStmt, 0);
+ szRoot = sqlite3_column_int64(pStmt, 1);
+ mxRoot = sqlite3_column_int64(pStmt, 2);
+ }
+ sqlite3_finalize(pStmt);
+
+ printf("Number of segments....................... %9d\n", nSeg+nRoot);
+ printf("Number of leaf segments.................. %9d\n", nSeg-nIdx);
+ printf("Number of index segments................. %9d\n", nIdx);
+ printf("Number of root segments.................. %9d\n", nRoot);
+ printf("Total size of all segments............... %9lld\n", szSeg+szRoot);
+ printf("Total size of all leaf segments.......... %9lld\n", szSeg-szIdx);
+ printf("Total size of all index segments......... %9lld\n", szIdx);
+ printf("Total size of all root segments.......... %9lld\n", szRoot);
+ if( nSeg>0 ){
+ printf("Average size of all segments............. %11.1f\n",
+ (double)(szSeg+szRoot)/(double)(nSeg+nRoot));
+ printf("Average size of leaf segments............ %11.1f\n",
+ (double)(szSeg-szIdx)/(double)(nSeg-nIdx));
+ }
+ if( nIdx>0 ){
+ printf("Average size of index segments........... %11.1f\n",
+ (double)szIdx/(double)nIdx);
+ }
+ if( nRoot>0 ){
+ printf("Average size of root segments............ %11.1f\n",
+ (double)szRoot/(double)nRoot);
+ }
+ mx = mxSeg;
+ if( mx<mxRoot ) mx = mxRoot;
+ printf("Maximum segment size..................... %9lld\n", mx);
+ printf("Maximum index segment size............... %9lld\n", mxIdx);
+ printf("Maximum root segment size................ %9lld\n", mxRoot);
+
+ pStmt = prepare(db, "PRAGMA page_size");
+ pgsz = 1024;
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ pgsz = sqlite3_column_int(pStmt, 0);
+ }
+ sqlite3_finalize(pStmt);
+ printf("Database page size....................... %9d\n", pgsz);
+ pStmt = prepare(db,
+ "SELECT count(*)"
+ " FROM '%q_segments' a JOIN '%q_segdir' b"
+ " WHERE a.blockid BETWEEN b.start_block AND b.leaves_end_block"
+ " AND length(a.block)>%d",
+ zTab, zTab, pgsz-45);
+ n = 0;
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ n = sqlite3_column_int(pStmt, 0);
+ }
+ sqlite3_finalize(pStmt);
+ nLeaf = nSeg - nIdx;
+ printf("Leaf segments larger than %5d bytes.... %9d %5.2f%%\n",
+ pgsz-45, n, n*100.0/nLeaf);
+
+ pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab);
+ mxLevel = 0;
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ mxLevel = sqlite3_column_int(pStmt, 0);
+ }
+ sqlite3_finalize(pStmt);
+
+ for(i=0; i<=mxLevel; i++){
+ pStmt = prepare(db,
+ "SELECT count(*), sum(len), avg(len), max(len), sum(len>%d),"
+ " count(distinct idx)"
+ " FROM (SELECT length(a.block) AS len, idx"
+ " FROM '%q_segments' a JOIN '%q_segdir' b"
+ " WHERE (a.blockid BETWEEN b.start_block"
+ " AND b.leaves_end_block)"
+ " AND (b.level%%1024)==%d)",
+ pgsz-45, zTab, zTab, i);
+ if( sqlite3_step(pStmt)==SQLITE_ROW
+ && (nLeaf = sqlite3_column_int(pStmt, 0))>0
+ ){
+ int nIdx = sqlite3_column_int(pStmt, 5);
+ sqlite3_int64 sz;
+ printf("For level %d:\n", i);
+ printf(" Number of indexes...................... %9d\n", nIdx);
+ printf(" Number of leaf segments................ %9d\n", nLeaf);
+ if( nIdx>1 ){
+ printf(" Average leaf segments per index........ %11.1f\n",
+ (double)nLeaf/(double)nIdx);
+ }
+ printf(" Total size of all leaf segments........ %9lld\n",
+ (sz = sqlite3_column_int64(pStmt, 1)));
+ printf(" Average size of leaf segments.......... %11.1f\n",
+ sqlite3_column_double(pStmt, 2));
+ if( nIdx>1 ){
+ printf(" Average leaf segment size per index.... %11.1f\n",
+ (double)sz/(double)nIdx);
+ }
+ printf(" Maximum leaf segment size.............. %9lld\n",
+ sqlite3_column_int64(pStmt, 3));
+ n = sqlite3_column_int(pStmt, 4);
+ printf(" Leaf segments larger than %5d bytes.. %9d %5.2f%%\n",
+ pgsz-45, n, n*100.0/nLeaf);
+ }
+ sqlite3_finalize(pStmt);
+ }
+}
+
+/*
+** Print a single "tree" line of the segdir map output.
+*/
+static void printTreeLine(sqlite3_int64 iLower, sqlite3_int64 iUpper){
+ printf(" tree %9lld", iLower);
+ if( iUpper>iLower ){
+ printf(" thru %9lld (%lld blocks)", iUpper, iUpper-iLower+1);
+ }
+ printf("\n");
+}
+
+/*
+** Check to see if the block of a %_segments entry is NULL.
+*/
+static int isNullSegment(sqlite3 *db, const char *zTab, sqlite3_int64 iBlockId){
+ sqlite3_stmt *pStmt;
+ int rc = 1;
+
+ pStmt = prepare(db, "SELECT block IS NULL FROM '%q_segments'"
+ " WHERE blockid=%lld", zTab, iBlockId);
+ if( sqlite3_step(pStmt)==SQLITE_ROW ){
+ rc = sqlite3_column_int(pStmt, 0);
+ }
+ sqlite3_finalize(pStmt);
+ return rc;
+}
+
+/*
+** Show a map of segments derived from the %_segdir table.
+*/
+static void showSegdirMap(sqlite3 *db, const char *zTab){
+ int mxIndex, iIndex;
+ sqlite3_stmt *pStmt = 0;
+ sqlite3_stmt *pStmt2 = 0;
+ int prevLevel;
+
+ pStmt = prepare(db, "SELECT max(level/1024) FROM '%q_segdir'", zTab);
+ if( sqlite3_step(pStmt)==SQLITE_ROW ){
+ mxIndex = sqlite3_column_int(pStmt, 0);
+ }else{
+ mxIndex = 0;
+ }
+ sqlite3_finalize(pStmt);
+
+ printf("Number of inverted indices............... %3d\n", mxIndex+1);
+ pStmt = prepare(db,
+ "SELECT level, idx, start_block, leaves_end_block, end_block, rowid"
+ " FROM '%q_segdir'"
+ " WHERE level/1024==?"
+ " ORDER BY level DESC, idx",
+ zTab);
+ pStmt2 = prepare(db,
+ "SELECT blockid FROM '%q_segments'"
+ " WHERE blockid BETWEEN ? AND ? ORDER BY blockid",
+ zTab);
+ for(iIndex=0; iIndex<=mxIndex; iIndex++){
+ if( mxIndex>0 ){
+ printf("**************************** Index %d "
+ "****************************\n", iIndex);
+ }
+ sqlite3_bind_int(pStmt, 1, iIndex);
+ prevLevel = -1;
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ int iLevel = sqlite3_column_int(pStmt, 0)%1024;
+ int iIdx = sqlite3_column_int(pStmt, 1);
+ sqlite3_int64 iStart = sqlite3_column_int64(pStmt, 2);
+ sqlite3_int64 iLEnd = sqlite3_column_int64(pStmt, 3);
+ sqlite3_int64 iEnd = sqlite3_column_int64(pStmt, 4);
+ char rtag[20];
+ if( iLevel!=prevLevel ){
+ printf("level %2d idx %2d", iLevel, iIdx);
+ prevLevel = iLevel;
+ }else{
+ printf(" idx %2d", iIdx);
+ }
+ sqlite3_snprintf(sizeof(rtag), rtag, "r%lld",
+ sqlite3_column_int64(pStmt,5));
+ printf(" root %9s\n", rtag);
+ if( iLEnd>iStart ){
+ sqlite3_int64 iLower, iPrev, iX;
+ if( iLEnd+1<=iEnd ){
+ sqlite3_bind_int64(pStmt2, 1, iLEnd+1);
+ sqlite3_bind_int64(pStmt2, 2, iEnd);
+ iLower = -1;
+ while( sqlite3_step(pStmt2)==SQLITE_ROW ){
+ iX = sqlite3_column_int64(pStmt2, 0);
+ if( iLower<0 ){
+ iLower = iPrev = iX;
+ }else if( iX==iPrev+1 ){
+ iPrev = iX;
+ }else{
+ printTreeLine(iLower, iPrev);
+ iLower = iPrev = iX;
+ }
+ }
+ sqlite3_reset(pStmt2);
+ if( iLower>=0 ){
+ if( iLower==iPrev && iLower==iEnd
+ && isNullSegment(db,zTab,iLower)
+ ){
+ printf(" null %9lld\n", iLower);
+ }else{
+ printTreeLine(iLower, iPrev);
+ }
+ }
+ }
+ printf(" leaves %9lld thru %9lld (%lld blocks)\n",
+ iStart, iLEnd, iLEnd - iStart + 1);
+ }
+ }
+ sqlite3_reset(pStmt);
+ }
+ sqlite3_finalize(pStmt);
+ sqlite3_finalize(pStmt2);
+}
+
+/*
+** Decode a single segment block and display the results on stdout.
+*/
+static void decodeSegment(
+ const unsigned char *aData, /* Content to print */
+ int nData /* Number of bytes of content */
+){
+ sqlite3_int64 iChild;
+ sqlite3_int64 iPrefix;
+ sqlite3_int64 nTerm;
+ sqlite3_int64 n;
+ sqlite3_int64 iDocsz;
+ int iHeight;
+ int i = 0;
+ int cnt = 0;
+ char zTerm[1000];
+
+ i += getVarint(aData, &n);
+ iHeight = (int)n;
+ printf("height: %d\n", iHeight);
+ if( iHeight>0 ){
+ i += getVarint(aData+i, &iChild);
+ printf("left-child: %lld\n", iChild);
+ }
+ while( i<nData ){
+ if( (cnt++)>0 ){
+ i += getVarint(aData+i, &iPrefix);
+ }else{
+ iPrefix = 0;
+ }
+ i += getVarint(aData+i, &nTerm);
+ if( iPrefix+nTerm+1 >= sizeof(zTerm) ){
+ fprintf(stderr, "term to long\n");
+ exit(1);
+ }
+ memcpy(zTerm+iPrefix, aData+i, nTerm);
+ zTerm[iPrefix+nTerm] = 0;
+ i += nTerm;
+ if( iHeight==0 ){
+ i += getVarint(aData+i, &iDocsz);
+ printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i);
+ i += iDocsz;
+ }else{
+ printf("term: %-25s child %lld\n", zTerm, ++iChild);
+ }
+ }
+}
+
+
+/*
+** Print a a blob as hex and ascii.
+*/
+static void printBlob(
+ const unsigned char *aData, /* Content to print */
+ int nData /* Number of bytes of content */
+){
+ int i, j;
+ const char *zOfstFmt;
+ const int perLine = 16;
+
+ if( (nData&~0xfff)==0 ){
+ zOfstFmt = " %03x: ";
+ }else if( (nData&~0xffff)==0 ){
+ zOfstFmt = " %04x: ";
+ }else if( (nData&~0xfffff)==0 ){
+ zOfstFmt = " %05x: ";
+ }else if( (nData&~0xffffff)==0 ){
+ zOfstFmt = " %06x: ";
+ }else{
+ zOfstFmt = " %08x: ";
+ }
+
+ for(i=0; i<nData; i += perLine){
+ fprintf(stdout, zOfstFmt, i);
+ for(j=0; j<perLine; j++){
+ if( i+j>nData ){
+ fprintf(stdout, " ");
+ }else{
+ fprintf(stdout,"%02x ", aData[i+j]);
+ }
+ }
+ for(j=0; j<perLine; j++){
+ if( i+j>nData ){
+ fprintf(stdout, " ");
+ }else{
+ fprintf(stdout,"%c", isprint(aData[i+j]) ? aData[i+j] : '.');
+ }
+ }
+ fprintf(stdout,"\n");
+ }
+}
+
+/*
+** Convert text to a 64-bit integer
+*/
+static sqlite3_int64 atoi64(const char *z){
+ sqlite3_int64 v = 0;
+ while( z[0]>='0' && z[0]<='9' ){
+ v = v*10 + z[0] - '0';
+ z++;
+ }
+ return v;
+}
+
+/*
+** Return a prepared statement which, when stepped, will return in its
+** first column the blob associated with segment zId. If zId begins with
+** 'r' then it is a rowid of a %_segdir entry. Otherwise it is a
+** %_segment entry.
+*/
+static sqlite3_stmt *prepareToGetSegment(
+ sqlite3 *db, /* The database */
+ const char *zTab, /* The FTS3/4 table name */
+ const char *zId /* ID of the segment to open */
+){
+ sqlite3_stmt *pStmt;
+ if( zId[0]=='r' ){
+ pStmt = prepare(db, "SELECT root FROM '%q_segdir' WHERE rowid=%lld",
+ zTab, atoi64(zId+1));
+ }else{
+ pStmt = prepare(db, "SELECT block FROM '%q_segments' WHERE blockid=%lld",
+ zTab, atoi64(zId));
+ }
+ return pStmt;
+}
+
+/*
+** Print the content of a segment or of the root of a segdir. The segment
+** or root is identified by azExtra[0]. If the first character of azExtra[0]
+** is 'r' then the remainder is the integer rowid of the %_segdir entry.
+** If the first character of azExtra[0] is not 'r' then, then all of
+** azExtra[0] is an integer which is the block number.
+**
+** If the --raw option is present in azExtra, then a hex dump is provided.
+** Otherwise a decoding is shown.
+*/
+static void showSegment(sqlite3 *db, const char *zTab){
+ const unsigned char *aData;
+ int nData;
+ sqlite3_stmt *pStmt;
+
+ pStmt = prepareToGetSegment(db, zTab, azExtra[0]);
+ if( sqlite3_step(pStmt)!=SQLITE_ROW ){
+ sqlite3_finalize(pStmt);
+ return;
+ }
+ nData = sqlite3_column_bytes(pStmt, 0);
+ aData = sqlite3_column_blob(pStmt, 0);
+ printf("Segment %s of size %d bytes:\n", azExtra[0], nData);
+ if( findOption("raw", 0, 0)!=0 ){
+ printBlob(aData, nData);
+ }else{
+ decodeSegment(aData, nData);
+ }
+ sqlite3_finalize(pStmt);
+}
+
+/*
+** Decode a single doclist and display the results on stdout.
+*/
+static void decodeDoclist(
+ const unsigned char *aData, /* Content to print */
+ int nData /* Number of bytes of content */
+){
+ sqlite3_int64 iPrevDocid = 0;
+ sqlite3_int64 iDocid;
+ sqlite3_int64 iPos;
+ sqlite3_int64 iPrevPos = 0;
+ sqlite3_int64 iCol;
+ int i = 0;
+
+ while( i<nData ){
+ i += getVarint(aData+i, &iDocid);
+ printf("docid %lld col0", iDocid+iPrevDocid);
+ iPrevDocid += iDocid;
+ iPrevPos = 0;
+ while( 1 ){
+ i += getVarint(aData+i, &iPos);
+ if( iPos==1 ){
+ i += getVarint(aData+i, &iCol);
+ printf(" col%lld", iCol);
+ iPrevPos = 0;
+ }else if( iPos==0 ){
+ printf("\n");
+ break;
+ }else{
+ iPrevPos += iPos - 2;
+ printf(" %lld", iPrevPos);
+ }
+ }
+ }
+}
+
+
+/*
+** Print the content of a doclist. The segment or segdir-root is
+** identified by azExtra[0]. If the first character of azExtra[0]
+** is 'r' then the remainder is the integer rowid of the %_segdir entry.
+** If the first character of azExtra[0] is not 'r' then, then all of
+** azExtra[0] is an integer which is the block number. The offset
+** into the segment is identified by azExtra[1]. The size of the doclist
+** is azExtra[2].
+**
+** If the --raw option is present in azExtra, then a hex dump is provided.
+** Otherwise a decoding is shown.
+*/
+static void showDoclist(sqlite3 *db, const char *zTab){
+ const unsigned char *aData;
+ sqlite3_int64 offset, nData;
+ sqlite3_stmt *pStmt;
+
+ offset = atoi64(azExtra[1]);
+ nData = atoi64(azExtra[2]);
+ pStmt = prepareToGetSegment(db, zTab, azExtra[0]);
+ if( sqlite3_step(pStmt)!=SQLITE_ROW ){
+ sqlite3_finalize(pStmt);
+ return;
+ }
+ aData = sqlite3_column_blob(pStmt, 0);
+ printf("Doclist at %s offset %lld of size %lld bytes:\n",
+ azExtra[0], offset, nData);
+ if( findOption("raw", 0, 0)!=0 ){
+ printBlob(aData+offset, nData);
+ }else{
+ decodeDoclist(aData+offset, nData);
+ }
+ sqlite3_finalize(pStmt);
+}
+
+/*
+** Show the top N largest segments
+*/
+static void listBigSegments(sqlite3 *db, const char *zTab){
+ int nTop, i;
+ sqlite3_stmt *pStmt;
+ sqlite3_int64 sz;
+ sqlite3_int64 id;
+
+ nTop = atoi(findOption("top", 1, "25"));
+ printf("The %d largest segments:\n", nTop);
+ pStmt = prepare(db,
+ "SELECT blockid, length(block) AS len FROM '%q_segments'"
+ " ORDER BY 2 DESC, 1"
+ " LIMIT %d", zTab, nTop);
+ i = 0;
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ i++;
+ id = sqlite3_column_int64(pStmt, 0);
+ sz = sqlite3_column_int64(pStmt, 1);
+ printf(" %2d. %9lld size %lld\n", i, id, sz);
+ }
+ sqlite3_finalize(pStmt);
+}
+
+
+
+static void usage(const char *argv0){
+ fprintf(stderr, "Usage: %s DATABASE\n"
+ " or: %s DATABASE FTS3TABLE ARGS...\n", argv0, argv0);
+ fprintf(stderr,
+ "ARGS:\n"
+ " big-segments [--top N] show the largest segments\n"
+ " doclist BLOCKID OFFSET SIZE [--raw] Decode a doclist\n"
+ " schema FTS table schema\n"
+ " segdir directory of segments\n"
+ " segment BLOCKID [--raw] content of a segment\n"
+ " segment-stats info on segment sizes\n"
+ " stat the %%_stat table\n"
+ " vocabulary [--top N] document vocabulary\n"
+ );
+ exit(1);
+}
+
+int main(int argc, char **argv){
+ sqlite3 *db;
+ int rc;
+ const char *zTab;
+ const char *zCmd;
+
+ if( argc<2 ) usage(argv[0]);
+ rc = sqlite3_open(argv[1], &db);
+ if( rc ){
+ fprintf(stderr, "Cannot open %s\n", argv[1]);
+ exit(1);
+ }
+ if( argc==2 ){
+ sqlite3_stmt *pStmt;
+ int cnt = 0;
+ pStmt = prepare(db, "SELECT b.sql"
+ " FROM sqlite_master a, sqlite_master b"
+ " WHERE a.name GLOB '*_segdir'"
+ " AND b.name=substr(a.name,1,length(a.name)-7)"
+ " ORDER BY 1");
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ cnt++;
+ printf("%s;\n", sqlite3_column_text(pStmt, 0));
+ }
+ sqlite3_finalize(pStmt);
+ if( cnt==0 ){
+ printf("/* No FTS3/4 tables found in database %s */\n", argv[1]);
+ }
+ return 0;
+ }
+ if( argc<4 ) usage(argv[0]);
+ zTab = argv[2];
+ zCmd = argv[3];
+ nExtra = argc-4;
+ azExtra = argv+4;
+ if( strcmp(zCmd,"big-segments")==0 ){
+ listBigSegments(db, zTab);
+ }else if( strcmp(zCmd,"doclist")==0 ){
+ if( argc<7 ) usage(argv[0]);
+ showDoclist(db, zTab);
+ }else if( strcmp(zCmd,"schema")==0 ){
+ showSchema(db, zTab);
+ }else if( strcmp(zCmd,"segdir")==0 ){
+ showSegdirMap(db, zTab);
+ }else if( strcmp(zCmd,"segment")==0 ){
+ if( argc<5 ) usage(argv[0]);
+ showSegment(db, zTab);
+ }else if( strcmp(zCmd,"segment-stats")==0 ){
+ showSegmentStats(db, zTab);
+ }else if( strcmp(zCmd,"stat")==0 ){
+ showStat(db, zTab);
+ }else if( strcmp(zCmd,"vocabulary")==0 ){
+ showVocabulary(db, zTab);
+ }else{
+ usage(argv[0]);
+ }
+ return 0;
+}
diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index 884482e..d6cdde9 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -183,6 +183,19 @@ struct Rtree {
#define RTREE_COORD_INT32 1
/*
+** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will
+** only deal with integer coordinates. No floating point operations
+** will be done.
+*/
+#ifdef SQLITE_RTREE_INT_ONLY
+ typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */
+ typedef int RtreeValue; /* Low accuracy coordinate */
+#else
+ typedef double RtreeDValue; /* High accuracy coordinate */
+ typedef float RtreeValue; /* Low accuracy coordinate */
+#endif
+
+/*
** The minimum number of cells allowed for a node is a third of the
** maximum. In Gutman's notation:
**
@@ -217,20 +230,25 @@ struct RtreeCursor {
};
union RtreeCoord {
- float f;
+ RtreeValue f;
int i;
};
/*
** The argument is an RtreeCoord. Return the value stored within the RtreeCoord
-** formatted as a double. This macro assumes that local variable pRtree points
-** to the Rtree structure associated with the RtreeCoord.
+** formatted as a RtreeDValue (double or int64). This macro assumes that local
+** variable pRtree points to the Rtree structure associated with the
+** RtreeCoord.
*/
-#define DCOORD(coord) ( \
- (pRtree->eCoordType==RTREE_COORD_REAL32) ? \
- ((double)coord.f) : \
- ((double)coord.i) \
-)
+#ifdef SQLITE_RTREE_INT_ONLY
+# define DCOORD(coord) ((RtreeDValue)coord.i)
+#else
+# define DCOORD(coord) ( \
+ (pRtree->eCoordType==RTREE_COORD_REAL32) ? \
+ ((double)coord.f) : \
+ ((double)coord.i) \
+ )
+#endif
/*
** A search constraint.
@@ -238,8 +256,8 @@ union RtreeCoord {
struct RtreeConstraint {
int iCoord; /* Index of constrained coordinate */
int op; /* Constraining operation */
- double rValue; /* Constraint value. */
- int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *);
+ RtreeDValue rValue; /* Constraint value. */
+ int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*);
sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */
};
@@ -287,10 +305,10 @@ struct RtreeCell {
*/
struct RtreeMatchArg {
u32 magic; /* Always RTREE_GEOMETRY_MAGIC */
- int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *);
+ int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue*, int *);
void *pContext;
int nParam;
- double aParam[1];
+ RtreeDValue aParam[1];
};
/*
@@ -302,7 +320,7 @@ struct RtreeMatchArg {
** the geometry callback function).
*/
struct RtreeGeomCallback {
- int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *);
+ int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*);
void *pContext;
};
@@ -868,7 +886,7 @@ static int testRtreeGeom(
int *pbRes /* OUT: Test result */
){
int i;
- double aCoord[RTREE_MAX_DIMENSIONS*2];
+ RtreeDValue aCoord[RTREE_MAX_DIMENSIONS*2];
int nCoord = pRtree->nDim*2;
assert( pConstraint->op==RTREE_MATCH );
@@ -898,8 +916,8 @@ static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){
nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell);
for(ii=0; bRes==0 && ii<pCursor->nConstraint; ii++){
RtreeConstraint *p = &pCursor->aConstraint[ii];
- double cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]);
- double cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]);
+ RtreeDValue cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]);
+ RtreeDValue cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]);
assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE
|| p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH
@@ -951,7 +969,7 @@ static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){
nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell);
for(ii=0; ii<pCursor->nConstraint; ii++){
RtreeConstraint *p = &pCursor->aConstraint[ii];
- double coord = DCOORD(cell.aCoord[p->iCoord]);
+ RtreeDValue coord = DCOORD(cell.aCoord[p->iCoord]);
int res;
assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE
|| p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH
@@ -1149,9 +1167,12 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
}else{
RtreeCoord c;
nodeGetCoord(pRtree, pCsr->pNode, pCsr->iCell, i-1, &c);
+#ifndef SQLITE_RTREE_INT_ONLY
if( pRtree->eCoordType==RTREE_COORD_REAL32 ){
sqlite3_result_double(ctx, c.f);
- }else{
+ }else
+#endif
+ {
assert( pRtree->eCoordType==RTREE_COORD_INT32 );
sqlite3_result_int(ctx, c.i);
}
@@ -1193,12 +1214,12 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){
int nBlob;
/* Check that value is actually a blob. */
- if( !sqlite3_value_type(pValue)==SQLITE_BLOB ) return SQLITE_ERROR;
+ if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR;
/* Check that the blob is roughly the right size. */
nBlob = sqlite3_value_bytes(pValue);
if( nBlob<(int)sizeof(RtreeMatchArg)
- || ((nBlob-sizeof(RtreeMatchArg))%sizeof(double))!=0
+ || ((nBlob-sizeof(RtreeMatchArg))%sizeof(RtreeDValue))!=0
){
return SQLITE_ERROR;
}
@@ -1212,7 +1233,7 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){
memcpy(p, sqlite3_value_blob(pValue), nBlob);
if( p->magic!=RTREE_GEOMETRY_MAGIC
- || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double))
+ || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(RtreeDValue))
){
sqlite3_free(pGeom);
return SQLITE_ERROR;
@@ -1284,7 +1305,11 @@ static int rtreeFilter(
break;
}
}else{
+#ifdef SQLITE_RTREE_INT_ONLY
+ p->rValue = sqlite3_value_int64(argv[ii]);
+#else
p->rValue = sqlite3_value_double(argv[ii]);
+#endif
}
}
}
@@ -1418,11 +1443,11 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
/*
** Return the N-dimensional volumn of the cell stored in *p.
*/
-static float cellArea(Rtree *pRtree, RtreeCell *p){
- float area = 1.0;
+static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){
+ RtreeDValue area = (RtreeDValue)1;
int ii;
for(ii=0; ii<(pRtree->nDim*2); ii+=2){
- area = (float)(area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])));
+ area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])));
}
return area;
}
@@ -1431,11 +1456,11 @@ static float cellArea(Rtree *pRtree, RtreeCell *p){
** Return the margin length of cell p. The margin length is the sum
** of the objects size in each dimension.
*/
-static float cellMargin(Rtree *pRtree, RtreeCell *p){
- float margin = 0.0;
+static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){
+ RtreeDValue margin = (RtreeDValue)0;
int ii;
for(ii=0; ii<(pRtree->nDim*2); ii+=2){
- margin += (float)(DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]));
+ margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]));
}
return margin;
}
@@ -1480,8 +1505,8 @@ static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){
/*
** Return the amount cell p would grow by if it were unioned with pCell.
*/
-static float cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){
- float area;
+static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){
+ RtreeDValue area;
RtreeCell cell;
memcpy(&cell, p, sizeof(RtreeCell));
area = cellArea(pRtree, &cell);
@@ -1490,7 +1515,7 @@ static float cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){
}
#if VARIANT_RSTARTREE_CHOOSESUBTREE || VARIANT_RSTARTREE_SPLIT
-static float cellOverlap(
+static RtreeDValue cellOverlap(
Rtree *pRtree,
RtreeCell *p,
RtreeCell *aCell,
@@ -1498,7 +1523,7 @@ static float cellOverlap(
int iExclude
){
int ii;
- float overlap = 0.0;
+ RtreeDValue overlap = 0.0;
for(ii=0; ii<nCell; ii++){
#if VARIANT_RSTARTREE_CHOOSESUBTREE
if( ii!=iExclude )
@@ -1508,10 +1533,9 @@ static float cellOverlap(
#endif
{
int jj;
- float o = 1.0;
+ RtreeDValue o = (RtreeDValue)1;
for(jj=0; jj<(pRtree->nDim*2); jj+=2){
- double x1;
- double x2;
+ RtreeDValue x1, x2;
x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj]));
x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1]));
@@ -1520,7 +1544,7 @@ static float cellOverlap(
o = 0.0;
break;
}else{
- o = o * (float)(x2-x1);
+ o = o * (x2-x1);
}
}
overlap += o;
@@ -1531,7 +1555,7 @@ static float cellOverlap(
#endif
#if VARIANT_RSTARTREE_CHOOSESUBTREE
-static float cellOverlapEnlargement(
+static RtreeDValue cellOverlapEnlargement(
Rtree *pRtree,
RtreeCell *p,
RtreeCell *pInsert,
@@ -1539,12 +1563,11 @@ static float cellOverlapEnlargement(
int nCell,
int iExclude
){
- double before;
- double after;
+ RtreeDValue before, after;
before = cellOverlap(pRtree, p, aCell, nCell, iExclude);
cellUnion(pRtree, p, pInsert);
after = cellOverlap(pRtree, p, aCell, nCell, iExclude);
- return (float)(after-before);
+ return (after-before);
}
#endif
@@ -1568,11 +1591,11 @@ static int ChooseLeaf(
int iCell;
sqlite3_int64 iBest = 0;
- float fMinGrowth = 0.0;
- float fMinArea = 0.0;
+ RtreeDValue fMinGrowth = 0.0;
+ RtreeDValue fMinArea = 0.0;
#if VARIANT_RSTARTREE_CHOOSESUBTREE
- float fMinOverlap = 0.0;
- float overlap;
+ RtreeDValue fMinOverlap = 0.0;
+ RtreeDValue overlap;
#endif
int nCell = NCELL(pNode);
@@ -1603,8 +1626,8 @@ static int ChooseLeaf(
*/
for(iCell=0; iCell<nCell; iCell++){
int bBest = 0;
- float growth;
- float area;
+ RtreeDValue growth;
+ RtreeDValue area;
nodeGetCell(pRtree, pNode, iCell, &cell);
growth = cellGrowth(pRtree, &cell, pCell);
area = cellArea(pRtree, &cell);
@@ -1731,7 +1754,7 @@ static void LinearPickSeeds(
int i;
int iLeftSeed = 0;
int iRightSeed = 1;
- float maxNormalInnerWidth = 0.0;
+ RtreeDValue maxNormalInnerWidth = (RtreeDValue)0;
/* Pick two "seed" cells from the array of cells. The algorithm used
** here is the LinearPickSeeds algorithm from Gutman[1984]. The
@@ -1739,18 +1762,18 @@ static void LinearPickSeeds(
** variables iLeftSeek and iRightSeed.
*/
for(i=0; i<pRtree->nDim; i++){
- float x1 = DCOORD(aCell[0].aCoord[i*2]);
- float x2 = DCOORD(aCell[0].aCoord[i*2+1]);
- float x3 = x1;
- float x4 = x2;
+ RtreeDValue x1 = DCOORD(aCell[0].aCoord[i*2]);
+ RtreeDValue x2 = DCOORD(aCell[0].aCoord[i*2+1]);
+ RtreeDValue x3 = x1;
+ RtreeDValue x4 = x2;
int jj;
int iCellLeft = 0;
int iCellRight = 0;
for(jj=1; jj<nCell; jj++){
- float left = DCOORD(aCell[jj].aCoord[i*2]);
- float right = DCOORD(aCell[jj].aCoord[i*2+1]);
+ RtreeDValue left = DCOORD(aCell[jj].aCoord[i*2]);
+ RtreeDValue right = DCOORD(aCell[jj].aCoord[i*2+1]);
if( left<x1 ) x1 = left;
if( right>x4 ) x4 = right;
@@ -1765,7 +1788,7 @@ static void LinearPickSeeds(
}
if( x4!=x1 ){
- float normalwidth = (x3 - x2) / (x4 - x1);
+ RtreeDValue normalwidth = (x3 - x2) / (x4 - x1);
if( normalwidth>maxNormalInnerWidth ){
iLeftSeed = iCellLeft;
iRightSeed = iCellRight;
@@ -1794,13 +1817,13 @@ static RtreeCell *QuadraticPickNext(
#define FABS(a) ((a)<0.0?-1.0*(a):(a))
int iSelect = -1;
- float fDiff;
+ RtreeDValue fDiff;
int ii;
for(ii=0; ii<nCell; ii++){
if( aiUsed[ii]==0 ){
- float left = cellGrowth(pRtree, pLeftBox, &aCell[ii]);
- float right = cellGrowth(pRtree, pLeftBox, &aCell[ii]);
- float diff = FABS(right-left);
+ RtreeDValue left = cellGrowth(pRtree, pLeftBox, &aCell[ii]);
+ RtreeDValue right = cellGrowth(pRtree, pLeftBox, &aCell[ii]);
+ RtreeDValue diff = FABS(right-left);
if( iSelect<0 || diff>fDiff ){
fDiff = diff;
iSelect = ii;
@@ -1827,13 +1850,13 @@ static void QuadraticPickSeeds(
int iLeftSeed = 0;
int iRightSeed = 1;
- float fWaste = 0.0;
+ RtreeDValue fWaste = 0.0;
for(ii=0; ii<nCell; ii++){
for(jj=ii+1; jj<nCell; jj++){
- float right = cellArea(pRtree, &aCell[jj]);
- float growth = cellGrowth(pRtree, &aCell[ii], &aCell[jj]);
- float waste = growth - right;
+ RtreeDValue right = cellArea(pRtree, &aCell[jj]);
+ RtreeDValue growth = cellGrowth(pRtree, &aCell[ii], &aCell[jj]);
+ RtreeDValue waste = growth - right;
if( waste>fWaste ){
iLeftSeed = ii;
@@ -1868,7 +1891,7 @@ static void QuadraticPickSeeds(
static void SortByDistance(
int *aIdx,
int nIdx,
- float *aDistance,
+ RtreeDValue *aDistance,
int *aSpare
){
if( nIdx>1 ){
@@ -1894,8 +1917,8 @@ static void SortByDistance(
aIdx[iLeft+iRight] = aLeft[iLeft];
iLeft++;
}else{
- float fLeft = aDistance[aLeft[iLeft]];
- float fRight = aDistance[aRight[iRight]];
+ RtreeDValue fLeft = aDistance[aLeft[iLeft]];
+ RtreeDValue fRight = aDistance[aRight[iRight]];
if( fLeft<fRight ){
aIdx[iLeft+iRight] = aLeft[iLeft];
iLeft++;
@@ -1911,8 +1934,8 @@ static void SortByDistance(
{
int jj;
for(jj=1; jj<nIdx; jj++){
- float left = aDistance[aIdx[jj-1]];
- float right = aDistance[aIdx[jj]];
+ RtreeDValue left = aDistance[aIdx[jj-1]];
+ RtreeDValue right = aDistance[aIdx[jj]];
assert( left<=right );
}
}
@@ -1955,10 +1978,10 @@ static void SortByDimension(
memcpy(aSpare, aLeft, sizeof(int)*nLeft);
aLeft = aSpare;
while( iLeft<nLeft || iRight<nRight ){
- double xleft1 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2]);
- double xleft2 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2+1]);
- double xright1 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2]);
- double xright2 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2+1]);
+ RtreeDValue xleft1 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2]);
+ RtreeDValue xleft2 = DCOORD(aCell[aLeft[iLeft]].aCoord[iDim*2+1]);
+ RtreeDValue xright1 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2]);
+ RtreeDValue xright2 = DCOORD(aCell[aRight[iRight]].aCoord[iDim*2+1]);
if( (iLeft!=nLeft) && ((iRight==nRight)
|| (xleft1<xright1)
|| (xleft1==xright1 && xleft2<xright2)
@@ -1976,10 +1999,10 @@ static void SortByDimension(
{
int jj;
for(jj=1; jj<nIdx; jj++){
- float xleft1 = aCell[aIdx[jj-1]].aCoord[iDim*2];
- float xleft2 = aCell[aIdx[jj-1]].aCoord[iDim*2+1];
- float xright1 = aCell[aIdx[jj]].aCoord[iDim*2];
- float xright2 = aCell[aIdx[jj]].aCoord[iDim*2+1];
+ RtreeDValue xleft1 = aCell[aIdx[jj-1]].aCoord[iDim*2];
+ RtreeDValue xleft2 = aCell[aIdx[jj-1]].aCoord[iDim*2+1];
+ RtreeDValue xright1 = aCell[aIdx[jj]].aCoord[iDim*2];
+ RtreeDValue xright2 = aCell[aIdx[jj]].aCoord[iDim*2+1];
assert( xleft1<=xright1 && (xleft1<xright1 || xleft2<=xright2) );
}
}
@@ -2006,7 +2029,7 @@ static int splitNodeStartree(
int iBestDim = 0;
int iBestSplit = 0;
- float fBestMargin = 0.0;
+ RtreeDValue fBestMargin = 0.0;
int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int));
@@ -2027,9 +2050,9 @@ static int splitNodeStartree(
}
for(ii=0; ii<pRtree->nDim; ii++){
- float margin = 0.0;
- float fBestOverlap = 0.0;
- float fBestArea = 0.0;
+ RtreeDValue margin = 0.0;
+ RtreeDValue fBestOverlap = 0.0;
+ RtreeDValue fBestArea = 0.0;
int iBestLeft = 0;
int nLeft;
@@ -2041,8 +2064,8 @@ static int splitNodeStartree(
RtreeCell left;
RtreeCell right;
int kk;
- float overlap;
- float area;
+ RtreeDValue overlap;
+ RtreeDValue area;
memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell));
memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell));
@@ -2125,7 +2148,7 @@ static int splitNodeGuttman(
for(i=nCell-2; i>0; i--){
RtreeCell *pNext;
pNext = PickNext(pRtree, aCell, nCell, pBboxLeft, pBboxRight, aiUsed);
- float diff =
+ RtreeDValue diff =
cellGrowth(pRtree, pBboxLeft, pNext) -
cellGrowth(pRtree, pBboxRight, pNext)
;
@@ -2458,32 +2481,34 @@ static int Reinsert(
int *aOrder;
int *aSpare;
RtreeCell *aCell;
- float *aDistance;
+ RtreeDValue *aDistance;
int nCell;
- float aCenterCoord[RTREE_MAX_DIMENSIONS];
+ RtreeDValue aCenterCoord[RTREE_MAX_DIMENSIONS];
int iDim;
int ii;
int rc = SQLITE_OK;
+ int n;
- memset(aCenterCoord, 0, sizeof(float)*RTREE_MAX_DIMENSIONS);
+ memset(aCenterCoord, 0, sizeof(RtreeDValue)*RTREE_MAX_DIMENSIONS);
nCell = NCELL(pNode)+1;
+ n = (nCell+1)&(~1);
/* Allocate the buffers used by this operation. The allocation is
** relinquished before this function returns.
*/
- aCell = (RtreeCell *)sqlite3_malloc(nCell * (
- sizeof(RtreeCell) + /* aCell array */
- sizeof(int) + /* aOrder array */
- sizeof(int) + /* aSpare array */
- sizeof(float) /* aDistance array */
+ aCell = (RtreeCell *)sqlite3_malloc(n * (
+ sizeof(RtreeCell) + /* aCell array */
+ sizeof(int) + /* aOrder array */
+ sizeof(int) + /* aSpare array */
+ sizeof(RtreeDValue) /* aDistance array */
));
if( !aCell ){
return SQLITE_NOMEM;
}
- aOrder = (int *)&aCell[nCell];
- aSpare = (int *)&aOrder[nCell];
- aDistance = (float *)&aSpare[nCell];
+ aOrder = (int *)&aCell[n];
+ aSpare = (int *)&aOrder[n];
+ aDistance = (RtreeDValue *)&aSpare[n];
for(ii=0; ii<nCell; ii++){
if( ii==(nCell-1) ){
@@ -2493,19 +2518,19 @@ static int Reinsert(
}
aOrder[ii] = ii;
for(iDim=0; iDim<pRtree->nDim; iDim++){
- aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2]);
- aCenterCoord[iDim] += (float)DCOORD(aCell[ii].aCoord[iDim*2+1]);
+ aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]);
+ aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]);
}
}
for(iDim=0; iDim<pRtree->nDim; iDim++){
- aCenterCoord[iDim] = (float)(aCenterCoord[iDim]/((float)nCell*2.0));
+ aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2));
}
for(ii=0; ii<nCell; ii++){
aDistance[ii] = 0.0;
for(iDim=0; iDim<pRtree->nDim; iDim++){
- float coord = (float)(DCOORD(aCell[ii].aCoord[iDim*2+1]) -
- DCOORD(aCell[ii].aCoord[iDim*2]));
+ RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) -
+ DCOORD(aCell[ii].aCoord[iDim*2]));
aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]);
}
}
@@ -2747,16 +2772,19 @@ static int rtreeUpdate(
/* Populate the cell.aCoord[] array. The first coordinate is azData[3]. */
assert( nData==(pRtree->nDim*2 + 3) );
+#ifndef SQLITE_RTREE_INT_ONLY
if( pRtree->eCoordType==RTREE_COORD_REAL32 ){
for(ii=0; ii<(pRtree->nDim*2); ii+=2){
- cell.aCoord[ii].f = (float)sqlite3_value_double(azData[ii+3]);
- cell.aCoord[ii+1].f = (float)sqlite3_value_double(azData[ii+4]);
+ cell.aCoord[ii].f = (RtreeValue)sqlite3_value_double(azData[ii+3]);
+ cell.aCoord[ii+1].f = (RtreeValue)sqlite3_value_double(azData[ii+4]);
if( cell.aCoord[ii].f>cell.aCoord[ii+1].f ){
rc = SQLITE_CONSTRAINT;
goto constraint;
}
}
- }else{
+ }else
+#endif
+ {
for(ii=0; ii<(pRtree->nDim*2); ii+=2){
cell.aCoord[ii].i = sqlite3_value_int(azData[ii+3]);
cell.aCoord[ii+1].i = sqlite3_value_int(azData[ii+4]);
@@ -3056,8 +3084,8 @@ static int rtreeInit(
sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
/* Allocate the sqlite3_vtab structure */
- nDb = strlen(argv[1]);
- nName = strlen(argv[2]);
+ nDb = (int)strlen(argv[1]);
+ nName = (int)strlen(argv[2]);
pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2);
if( !pRtree ){
return SQLITE_NOMEM;
@@ -3152,10 +3180,16 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){
nodeGetCell(&tree, &node, ii, &cell);
sqlite3_snprintf(512-nCell,&zCell[nCell],"%lld", cell.iRowid);
- nCell = strlen(zCell);
+ nCell = (int)strlen(zCell);
for(jj=0; jj<tree.nDim*2; jj++){
- sqlite3_snprintf(512-nCell,&zCell[nCell]," %f",(double)cell.aCoord[jj].f);
- nCell = strlen(zCell);
+#ifndef SQLITE_RTREE_INT_ONLY
+ sqlite3_snprintf(512-nCell,&zCell[nCell], " %f",
+ (double)cell.aCoord[jj].f);
+#else
+ sqlite3_snprintf(512-nCell,&zCell[nCell], " %d",
+ cell.aCoord[jj].i);
+#endif
+ nCell = (int)strlen(zCell);
}
if( zText ){
@@ -3196,7 +3230,11 @@ int sqlite3RtreeInit(sqlite3 *db){
rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0);
}
if( rc==SQLITE_OK ){
+#ifdef SQLITE_RTREE_INT_ONLY
+ void *c = (void *)RTREE_COORD_INT32;
+#else
void *c = (void *)RTREE_COORD_REAL32;
+#endif
rc = sqlite3_create_module_v2(db, "rtree", &rtreeModule, c, 0);
}
if( rc==SQLITE_OK ){
@@ -3230,7 +3268,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){
RtreeMatchArg *pBlob;
int nBlob;
- nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(double);
+ nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue);
pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob);
if( !pBlob ){
sqlite3_result_error_nomem(ctx);
@@ -3241,7 +3279,11 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){
pBlob->pContext = pGeomCtx->pContext;
pBlob->nParam = nArg;
for(i=0; i<nArg; i++){
+#ifdef SQLITE_RTREE_INT_ONLY
+ pBlob->aParam[i] = sqlite3_value_int64(aArg[i]);
+#else
pBlob->aParam[i] = sqlite3_value_double(aArg[i]);
+#endif
}
sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free);
}
@@ -3253,7 +3295,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){
int sqlite3_rtree_geometry_callback(
sqlite3 *db,
const char *zGeom,
- int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *),
+ int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue *, int *),
void *pContext
){
RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */
diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test
index 583b028..e3c7d68 100644
--- a/ext/rtree/rtree1.test
+++ b/ext/rtree/rtree1.test
@@ -104,6 +104,18 @@ for {set nCol 1} {$nCol<[llength $cols]} {incr nCol} {
catchsql { DROP TABLE t1 }
}
+# Like execsql except display output as integer where that can be
+# done without loss of information.
+#
+proc execsql_intout {sql} {
+ set out {}
+ foreach term [execsql $sql] {
+ regsub {\.0$} $term {} term
+ lappend out $term
+ }
+ return $out
+}
+
# Test that it is possible to open an existing database that contains
# r-tree tables.
#
@@ -117,8 +129,8 @@ do_test rtree-1.4.1 {
do_test rtree-1.4.2 {
db close
sqlite3 db test.db
- execsql { SELECT * FROM t1 ORDER BY ii }
-} {1 5.0 10.0 2 15.0 20.0}
+ execsql_intout { SELECT * FROM t1 ORDER BY ii }
+} {1 5 10 2 15 20}
do_test rtree-1.4.3 {
execsql { DROP TABLE t1 }
} {}
@@ -127,12 +139,12 @@ do_test rtree-1.4.3 {
# column names.
#
do_test rtree-1.5.1 {
- execsql {
+ execsql_intout {
CREATE VIRTUAL TABLE t1 USING rtree("the key", "x dim.", "x2'dim");
INSERT INTO t1 VALUES(1, 2, 3);
SELECT "the key", "x dim.", "x2'dim" FROM t1;
}
-} {1 2.0 3.0}
+} {1 2 3}
do_test rtree-1.5.1 {
execsql { DROP TABLE t1 }
} {}
@@ -161,8 +173,8 @@ do_test rtree-2.1.1 {
do_test rtree-2.1.2 {
execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) }
- execsql { SELECT * FROM t1 }
-} {1 1.0 3.0 2.0 4.0}
+ execsql_intout { SELECT * FROM t1 }
+} {1 1 3 2 4}
do_test rtree-2.1.3 {
execsql { INSERT INTO t1 VALUES(NULL, 1, 3, 2, 4) }
execsql { SELECT rowid FROM t1 ORDER BY rowid }
@@ -201,17 +213,17 @@ do_test rtree-3.1.1 {
}
} {}
do_test rtree-3.1.2 {
- execsql {
+ execsql_intout {
INSERT INTO t1 VALUES(5, 1, 3, 2, 4);
SELECT * FROM t1;
}
-} {5 1.0 3.0 2.0 4.0}
+} {5 1 3 2 4}
do_test rtree-3.1.3 {
- execsql {
+ execsql_intout {
INSERT INTO t1 VALUES(6, 2, 6, 4, 8);
SELECT * FROM t1;
}
-} {5 1.0 3.0 2.0 4.0 6 2.0 6.0 4.0 8.0}
+} {5 1 3 2 4 6 2 6 4 8}
# Test the constraint on the coordinates (c[i]<=c[i+1] where (i%2==0)):
do_test rtree-3.2.1 {
@@ -228,25 +240,25 @@ do_test rtree-5.1.1 {
execsql { CREATE VIRTUAL TABLE t2 USING rtree(ii, x1, x2) }
} {}
do_test rtree-5.1.2 {
- execsql {
+ execsql_intout {
INSERT INTO t2 VALUES(1, 10, 20);
INSERT INTO t2 VALUES(2, 30, 40);
INSERT INTO t2 VALUES(3, 50, 60);
SELECT * FROM t2 ORDER BY ii;
}
-} {1 10.0 20.0 2 30.0 40.0 3 50.0 60.0}
+} {1 10 20 2 30 40 3 50 60}
do_test rtree-5.1.3 {
- execsql {
+ execsql_intout {
DELETE FROM t2 WHERE ii=2;
SELECT * FROM t2 ORDER BY ii;
}
-} {1 10.0 20.0 3 50.0 60.0}
+} {1 10 20 3 50 60}
do_test rtree-5.1.4 {
- execsql {
+ execsql_intout {
DELETE FROM t2 WHERE ii=1;
SELECT * FROM t2 ORDER BY ii;
}
-} {3 50.0 60.0}
+} {3 50 60}
do_test rtree-5.1.5 {
execsql {
DELETE FROM t2 WHERE ii=3;
@@ -264,16 +276,16 @@ do_test rtree-6.1.1 {
execsql { CREATE VIRTUAL TABLE t3 USING rtree(ii, x1, x2, y1, y2) }
} {}
do_test rtree-6.1.2 {
- execsql {
+ execsql_intout {
INSERT INTO t3 VALUES(1, 2, 3, 4, 5);
UPDATE t3 SET x2=5;
SELECT * FROM t3;
}
-} {1 2.0 5.0 4.0 5.0}
+} {1 2 5 4 5}
do_test rtree-6.1.3 {
execsql { UPDATE t3 SET ii = 2 }
- execsql { SELECT * FROM t3 }
-} {2 2.0 5.0 4.0 5.0}
+ execsql_intout { SELECT * FROM t3 }
+} {2 2 5 4 5}
#----------------------------------------------------------------------------
# Test cases rtree-7.* test rename operations.
@@ -286,29 +298,29 @@ do_test rtree-7.1.1 {
} {}
do_test rtree-7.1.2 {
execsql { ALTER TABLE t4 RENAME TO t5 }
- execsql { SELECT * FROM t5 }
-} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+ execsql_intout { SELECT * FROM t5 }
+} {1 2 3 4 5 6 7}
do_test rtree-7.1.3 {
db close
sqlite3 db test.db
- execsql { SELECT * FROM t5 }
-} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+ execsql_intout { SELECT * FROM t5 }
+} {1 2 3 4 5 6 7}
do_test rtree-7.1.4 {
execsql { ALTER TABLE t5 RENAME TO 'raisara "one"'''}
- execsql { SELECT * FROM "raisara ""one""'" }
-} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+ execsql_intout { SELECT * FROM "raisara ""one""'" }
+} {1 2 3 4 5 6 7}
do_test rtree-7.1.5 {
- execsql { SELECT * FROM 'raisara "one"''' }
-} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+ execsql_intout { SELECT * FROM 'raisara "one"''' }
+} {1 2 3 4 5 6 7}
do_test rtree-7.1.6 {
execsql { ALTER TABLE "raisara ""one""'" RENAME TO "abc 123" }
- execsql { SELECT * FROM "abc 123" }
-} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+ execsql_intout { SELECT * FROM "abc 123" }
+} {1 2 3 4 5 6 7}
do_test rtree-7.1.7 {
db close
sqlite3 db test.db
- execsql { SELECT * FROM "abc 123" }
-} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+ execsql_intout { SELECT * FROM "abc 123" }
+} {1 2 3 4 5 6 7}
# An error midway through a rename operation.
do_test rtree-7.2.1 {
@@ -318,8 +330,8 @@ do_test rtree-7.2.1 {
catchsql { ALTER TABLE "abc 123" RENAME TO t4 }
} {1 {SQL logic error or missing database}}
do_test rtree-7.2.2 {
- execsql { SELECT * FROM "abc 123" }
-} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+ execsql_intout { SELECT * FROM "abc 123" }
+} {1 2 3 4 5 6 7}
do_test rtree-7.2.3 {
execsql {
DROP TABLE t4_node;
@@ -330,13 +342,13 @@ do_test rtree-7.2.3 {
do_test rtree-7.2.4 {
db close
sqlite3 db test.db
- execsql { SELECT * FROM "abc 123" }
-} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+ execsql_intout { SELECT * FROM "abc 123" }
+} {1 2 3 4 5 6 7}
do_test rtree-7.2.5 {
execsql { DROP TABLE t4_rowid }
execsql { ALTER TABLE "abc 123" RENAME TO t4 }
- execsql { SELECT * FROM t4 }
-} {1 2.0 3.0 4.0 5.0 6.0 7.0}
+ execsql_intout { SELECT * FROM t4 }
+} {1 2 3 4 5 6 7}
#----------------------------------------------------------------------------
diff --git a/ext/rtree/rtree4.test b/ext/rtree/rtree4.test
index 708d335..a3872b0 100644
--- a/ext/rtree/rtree4.test
+++ b/ext/rtree/rtree4.test
@@ -27,21 +27,38 @@ if {[info exists G(isquick)] && $G(isquick)} {
set ::NROW 250
}
-# Return a floating point number between -X and X.
-#
-proc rand {X} {
- return [expr {int((rand()-0.5)*1024.0*$X)/512.0}]
-}
-
-# Return a positive floating point number less than or equal to X
-#
-proc randincr {X} {
- while 1 {
- set r [expr {int(rand()*$X*32.0)/32.0}]
- if {$r>0.0} {return $r}
+ifcapable !rtree_int_only {
+ # Return a floating point number between -X and X.
+ #
+ proc rand {X} {
+ return [expr {int((rand()-0.5)*1024.0*$X)/512.0}]
+ }
+
+ # Return a positive floating point number less than or equal to X
+ #
+ proc randincr {X} {
+ while 1 {
+ set r [expr {int(rand()*$X*32.0)/32.0}]
+ if {$r>0.0} {return $r}
+ }
+ }
+} else {
+ # For rtree_int_only, return an number between -X and X.
+ #
+ proc rand {X} {
+ return [expr {int((rand()-0.5)*2*$X)}]
+ }
+
+ # Return a positive integer less than or equal to X
+ #
+ proc randincr {X} {
+ while 1 {
+ set r [expr {int(rand()*$X)+1}]
+ if {$r>0} {return $r}
+ }
}
}
-
+
# Scramble the $inlist into a random order.
#
proc scramble {inlist} {
diff --git a/ext/rtree/rtree5.test b/ext/rtree/rtree5.test
index ea2946f..8990772 100644
--- a/ext/rtree/rtree5.test
+++ b/ext/rtree/rtree5.test
@@ -49,9 +49,11 @@ do_test rtree5-1.6 {
do_test rtree5-1.7 {
execsql { SELECT count(*) FROM t1 WHERE x1==5 }
} {1}
-do_test rtree5-1.8 {
- execsql { SELECT count(*) FROM t1 WHERE x1==5.2 }
-} {0}
+ifcapable !rtree_int_only {
+ do_test rtree5-1.8 {
+ execsql { SELECT count(*) FROM t1 WHERE x1==5.2 }
+ } {0}
+}
do_test rtree5-1.9 {
execsql { SELECT count(*) FROM t1 WHERE x1==5.0 }
} {1}
diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test
index ba0e53c..92edc8d 100644
--- a/ext/rtree/rtree6.test
+++ b/ext/rtree/rtree6.test
@@ -16,7 +16,7 @@ if {![info exists testdir]} {
}
source $testdir/tester.tcl
-ifcapable !rtree {
+ifcapable {!rtree || rtree_int_only} {
finish_test
return
}
diff --git a/ext/rtree/rtree7.test b/ext/rtree/rtree7.test
index 31dae0c..4eee4c2 100644
--- a/ext/rtree/rtree7.test
+++ b/ext/rtree/rtree7.test
@@ -24,6 +24,18 @@ ifcapable !rtree||!vacuum {
return
}
+# Like execsql except display output as integer where that can be
+# done without loss of information.
+#
+proc execsql_intout {sql} {
+ set out {}
+ foreach term [execsql $sql] {
+ regsub {\.0$} $term {} term
+ lappend out $term
+ }
+ return $out
+}
+
do_test rtree7-1.1 {
execsql {
PRAGMA page_size = 1024;
@@ -32,27 +44,27 @@ do_test rtree7-1.1 {
}
} {}
do_test rtree7-1.2 {
- execsql { SELECT * FROM rt }
-} {1 1.0 2.0 3.0 4.0}
+ execsql_intout { SELECT * FROM rt }
+} {1 1 2 3 4}
do_test rtree7-1.3 {
- execsql {
+ execsql_intout {
PRAGMA page_size = 2048;
VACUUM;
SELECT * FROM rt;
}
-} {1 1.0 2.0 3.0 4.0}
+} {1 1 2 3 4}
do_test rtree7-1.4 {
for {set i 2} {$i <= 51} {incr i} {
execsql { INSERT INTO rt VALUES($i, 1, 2, 3, 4) }
}
- execsql { SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt }
-} {51.0 102.0 153.0 204.0}
+ execsql_intout { SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt }
+} {51 102 153 204}
do_test rtree7-1.5 {
- execsql {
+ execsql_intout {
PRAGMA page_size = 512;
VACUUM;
SELECT sum(x1), sum(x2), sum(y1), sum(y2) FROM rt
}
-} {51.0 102.0 153.0 204.0}
+} {51 102 153 204}
finish_test
diff --git a/ext/rtree/rtree9.test b/ext/rtree/rtree9.test
index ddee277..6479516 100644
--- a/ext/rtree/rtree9.test
+++ b/ext/rtree/rtree9.test
@@ -17,6 +17,7 @@ if {![info exists testdir]} {
}
source $testdir/tester.tcl
ifcapable !rtree { finish_test ; return }
+ifcapable rtree_int_only { finish_test; return }
register_cube_geom db
diff --git a/ext/rtree/rtreeB.test b/ext/rtree/rtreeB.test
index 2756fce..7cb445c 100644
--- a/ext/rtree/rtreeB.test
+++ b/ext/rtree/rtreeB.test
@@ -18,17 +18,30 @@ if {![info exists testdir]} {
source $testdir/tester.tcl
ifcapable !rtree { finish_test ; return }
-do_test rtreeB-1.1 {
- db eval {
- CREATE VIRTUAL TABLE t1 USING rtree(ii, x0, y0, x1, y1);
- INSERT INTO t1 VALUES(1073741824, 0.0, 0.0, 100.0, 100.0);
- INSERT INTO t1 VALUES(2147483646, 0.0, 0.0, 200.0, 200.0);
- INSERT INTO t1 VALUES(4294967296, 0.0, 0.0, 300.0, 300.0);
- INSERT INTO t1 VALUES(8589934592, 20.0, 20.0, 150.0, 150.0);
- INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400);
- SELECT rtreenode(2, data) FROM t1_node;
- }
-} {{{1073741824 0.000000 0.000000 100.000000 100.000000} {2147483646 0.000000 0.000000 200.000000 200.000000} {4294967296 0.000000 0.000000 300.000000 300.000000} {8589934592 20.000000 20.000000 150.000000 150.000000} {9223372036854775807 150.000000 150.000000 400.000000 400.000000}}}
-
+ifcapable rtree_int_only {
+ do_test rtreeB-1.1-intonly {
+ db eval {
+ CREATE VIRTUAL TABLE t1 USING rtree(ii, x0, y0, x1, y1);
+ INSERT INTO t1 VALUES(1073741824, 0.0, 0.0, 100.0, 100.0);
+ INSERT INTO t1 VALUES(2147483646, 0.0, 0.0, 200.0, 200.0);
+ INSERT INTO t1 VALUES(4294967296, 0.0, 0.0, 300.0, 300.0);
+ INSERT INTO t1 VALUES(8589934592, 20.0, 20.0, 150.0, 150.0);
+ INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400);
+ SELECT rtreenode(2, data) FROM t1_node;
+ }
+ } {{{1073741824 0 0 100 100} {2147483646 0 0 200 200} {4294967296 0 0 300 300} {8589934592 20 20 150 150} {9223372036854775807 150 150 400 400}}}
+} else {
+ do_test rtreeB-1.1 {
+ db eval {
+ CREATE VIRTUAL TABLE t1 USING rtree(ii, x0, y0, x1, y1);
+ INSERT INTO t1 VALUES(1073741824, 0.0, 0.0, 100.0, 100.0);
+ INSERT INTO t1 VALUES(2147483646, 0.0, 0.0, 200.0, 200.0);
+ INSERT INTO t1 VALUES(4294967296, 0.0, 0.0, 300.0, 300.0);
+ INSERT INTO t1 VALUES(8589934592, 20.0, 20.0, 150.0, 150.0);
+ INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400);
+ SELECT rtreenode(2, data) FROM t1_node;
+ }
+ } {{{1073741824 0.000000 0.000000 100.000000 100.000000} {2147483646 0.000000 0.000000 200.000000 200.000000} {4294967296 0.000000 0.000000 300.000000 300.000000} {8589934592 20.000000 20.000000 150.000000 150.000000} {9223372036854775807 150.000000 150.000000 400.000000 400.000000}}}
+}
finish_test
diff --git a/ext/rtree/sqlite3rtree.h b/ext/rtree/sqlite3rtree.h
index cffb300..c849091 100644
--- a/ext/rtree/sqlite3rtree.h
+++ b/ext/rtree/sqlite3rtree.h
@@ -31,7 +31,11 @@ typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry;
int sqlite3_rtree_geometry_callback(
sqlite3 *db,
const char *zGeom,
- int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes),
+#ifdef SQLITE_RTREE_INT_ONLY
+ int (*xGeom)(sqlite3_rtree_geometry*, int n, sqlite3_int64 *a, int *pRes),
+#else
+ int (*xGeom)(sqlite3_rtree_geometry*, int n, double *a, int *pRes),
+#endif
void *pContext
);
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-18 03:02:38 +0000