summaryrefslogtreecommitdiff
path: root/ext/fts1/fulltext.c
diff options
context:
space:
mode:
Diffstat (limited to 'ext/fts1/fulltext.c')
-rw-r--r--ext/fts1/fulltext.c1496
1 files changed, 1496 insertions, 0 deletions
diff --git a/ext/fts1/fulltext.c b/ext/fts1/fulltext.c
new file mode 100644
index 0000000..e6034ba
--- /dev/null
+++ b/ext/fts1/fulltext.c
@@ -0,0 +1,1496 @@
+/* The author disclaims copyright to this source code.
+ *
+ * This is an SQLite module implementing full-text search.
+ */
+
+#include <assert.h>
+#if !defined(__APPLE__)
+#include <malloc.h>
+#else
+#include <stdlib.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "fulltext.h"
+#include "ft_hash.h"
+#include "tokenizer.h"
+#include "sqlite3.h"
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+
+/* utility functions */
+
+/* We encode variable-length integers in little-endian order using seven bits
+ * per byte as follows:
+**
+** KEY:
+** A = 0xxxxxxx 7 bits of data and one flag bit
+** B = 1xxxxxxx 7 bits of data and one flag bit
+**
+** 7 bits - A
+** 14 bits - BA
+** 21 bits - BBA
+** and so on.
+*/
+
+/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
+#define VARINT_MAX 10
+
+/* Write a 64-bit variable-length integer to memory starting at p[0].
+ * The length of data written will be between 1 and VARINT_MAX bytes.
+ * The number of bytes written is returned. */
+static int putVarint(char *p, sqlite_int64 v){
+ unsigned char *q = (unsigned char *) p;
+ sqlite_uint64 vu = v;
+ do{
+ *q++ = (unsigned char) ((vu & 0x7f) | 0x80);
+ vu >>= 7;
+ }while( vu!=0 );
+ q[-1] &= 0x7f; /* turn off high bit in final byte */
+ assert( q - (unsigned char *)p <= VARINT_MAX );
+ return (int) (q - (unsigned char *)p);
+}
+
+/* Read a 64-bit variable-length integer from memory starting at p[0].
+ * Return the number of bytes read, or 0 on error.
+ * The value is stored in *v. */
+static int getVarint(const char *p, sqlite_int64 *v){
+ const unsigned char *q = (const unsigned char *) p;
+ sqlite_uint64 x = 0, y = 1;
+ while( (*q & 0x80) == 0x80 ){
+ x += y * (*q++ & 0x7f);
+ y <<= 7;
+ if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */
+ assert( 0 );
+ return 0;
+ }
+ }
+ x += y * (*q++);
+ *v = (sqlite_int64) x;
+ return (int) (q - (unsigned char *)p);
+}
+
+static int getVarint32(const char *p, int *pi){
+ sqlite_int64 i;
+ int ret = getVarint(p, &i);
+ *pi = (int) i;
+ assert( *pi==i );
+ return ret;
+}
+
+/*** Document lists ***
+ *
+ * A document list holds a sorted list of varint-encoded document IDs.
+ *
+ * A doclist with type DL_POSITIONS_OFFSETS is stored like this:
+ *
+ * array {
+ * varint docid;
+ * array {
+ * varint position; (delta from previous position plus 1, or 0 for end)
+ * varint startOffset; (delta from previous startOffset)
+ * varint endOffset; (delta from startOffset)
+ * }
+ * }
+ *
+ * Here, array { X } means zero or more occurrences of X, adjacent in memory.
+ *
+ * A doclist with type DL_POSITIONS is like the above, but holds only docids
+ * and positions without offset information.
+ *
+ * A doclist with type DL_DOCIDS is like the above, but holds only docids
+ * without positions or offset information.
+ *
+ * On disk, every document list has positions and offsets, so we don't bother
+ * to serialize a doclist's type.
+ *
+ * We don't yet delta-encode document IDs; doing so will probably be a
+ * modest win.
+ *
+ * NOTE(shess) I've thought of a slightly (1%) better offset encoding.
+ * After the first offset, estimate the next offset by using the
+ * current token position and the previous token position and offset,
+ * offset to handle some variance. So the estimate would be
+ * (iPosition*w->iStartOffset/w->iPosition-64), which is delta-encoded
+ * as normal. Offsets more than 64 chars from the estimate are
+ * encoded as the delta to the previous start offset + 128. An
+ * additional tiny increment can be gained by using the end offset of
+ * the previous token to make the estimate a tiny bit more precise.
+*/
+
+typedef enum DocListType {
+ DL_DOCIDS, /* docids only */
+ DL_POSITIONS, /* docids + positions */
+ DL_POSITIONS_OFFSETS /* docids + positions + offsets */
+} DocListType;
+
+typedef struct DocList {
+ char *pData;
+ int nData;
+ DocListType iType;
+ int iLastPos; /* the last position written */
+ int iLastOffset; /* the last start offset written */
+} DocList;
+
+/* Initialize a new DocList to hold the given data. */
+static void docListInit(DocList *d, DocListType iType,
+ const char *pData, int nData){
+ d->nData = nData;
+ if( nData>0 ){
+ d->pData = malloc(nData);
+ memcpy(d->pData, pData, nData);
+ } else {
+ d->pData = NULL;
+ }
+ d->iType = iType;
+ d->iLastPos = 0;
+ d->iLastOffset = 0;
+}
+
+/* Create a new dynamically-allocated DocList. */
+static DocList *docListNew(DocListType iType){
+ DocList *d = (DocList *) malloc(sizeof(DocList));
+ docListInit(d, iType, 0, 0);
+ return d;
+}
+
+static void docListDestroy(DocList *d){
+ free(d->pData);
+#ifndef NDEBUG
+ memset(d, 0x55, sizeof(*d));
+#endif
+}
+
+static void docListDelete(DocList *d){
+ docListDestroy(d);
+ free(d);
+}
+
+static char *docListEnd(DocList *d){
+ return d->pData + d->nData;
+}
+
+/* Append a varint to a DocList's data. */
+static void appendVarint(DocList *d, sqlite_int64 i){
+ char c[VARINT_MAX];
+ int n = putVarint(c, i);
+ d->pData = realloc(d->pData, d->nData + n);
+ memcpy(d->pData + d->nData, c, n);
+ d->nData += n;
+}
+
+static void docListAddDocid(DocList *d, sqlite_int64 iDocid){
+ appendVarint(d, iDocid);
+ d->iLastPos = 0;
+}
+
+/* Add a position to the last position list in a doclist. */
+static void docListAddPos(DocList *d, int iPos){
+ assert( d->iType>=DL_POSITIONS );
+ appendVarint(d, iPos-d->iLastPos+1);
+ d->iLastPos = iPos;
+}
+
+static void docListAddPosOffset(DocList *d, int iPos,
+ int iStartOffset, int iEndOffset){
+ assert( d->iType==DL_POSITIONS_OFFSETS );
+ docListAddPos(d, iPos);
+ appendVarint(d, iStartOffset-d->iLastOffset);
+ d->iLastOffset = iStartOffset;
+ appendVarint(d, iEndOffset-iStartOffset);
+}
+
+/* Terminate the last position list in the given doclist. */
+static void docListAddEndPos(DocList *d){
+ appendVarint(d, 0);
+}
+
+typedef struct DocListReader {
+ DocList *pDoclist;
+ char *p;
+ int iLastPos; /* the last position read */
+} DocListReader;
+
+static void readerInit(DocListReader *r, DocList *pDoclist){
+ r->pDoclist = pDoclist;
+ if( pDoclist!=NULL ){
+ r->p = pDoclist->pData;
+ }
+ r->iLastPos = 0;
+}
+
+static int readerAtEnd(DocListReader *pReader){
+ return pReader->p >= docListEnd(pReader->pDoclist);
+}
+
+/* Peek at the next docid without advancing the read pointer. */
+static sqlite_int64 peekDocid(DocListReader *pReader){
+ sqlite_int64 ret;
+ assert( !readerAtEnd(pReader) );
+ getVarint(pReader->p, &ret);
+ return ret;
+}
+
+/* Read the next docid. */
+static sqlite_int64 readDocid(DocListReader *pReader){
+ sqlite_int64 ret;
+ assert( !readerAtEnd(pReader) );
+ pReader->p += getVarint(pReader->p, &ret);
+ pReader->iLastPos = 0;
+ return ret;
+}
+
+/* Read the next position from a position list.
+ * Returns the position, or -1 at the end of the list. */
+static int readPosition(DocListReader *pReader){
+ int i;
+ int iType = pReader->pDoclist->iType;
+ assert( iType>=DL_POSITIONS );
+ assert( !readerAtEnd(pReader) );
+
+ pReader->p += getVarint32(pReader->p, &i);
+ if( i==0 ){
+ pReader->iLastPos = -1;
+ return -1;
+ }
+ pReader->iLastPos += ((int) i)-1;
+ if( iType>=DL_POSITIONS_OFFSETS ){
+ /* Skip over offsets, ignoring them for now. */
+ int iStart, iEnd;
+ pReader->p += getVarint32(pReader->p, &iStart);
+ pReader->p += getVarint32(pReader->p, &iEnd);
+ }
+ return pReader->iLastPos;
+}
+
+/* Skip past the end of a position list. */
+static void skipPositionList(DocListReader *pReader){
+ while( readPosition(pReader)!=-1 )
+ ;
+}
+
+/* Skip over a docid, including its position list if the doclist has
+ * positions. */
+static void skipDocument(DocListReader *pReader){
+ readDocid(pReader);
+ if( pReader->pDoclist->iType >= DL_POSITIONS ){
+ skipPositionList(pReader);
+ }
+}
+
+static sqlite_int64 firstDocid(DocList *d){
+ DocListReader r;
+ readerInit(&r, d);
+ return readDocid(&r);
+}
+
+/* Doclist multi-tool. Pass pUpdate==NULL to delete the indicated docid;
+ * otherwise pUpdate, which must contain only the single docid [iDocid], is
+ * inserted (if not present) or updated (if already present). */
+static int docListUpdate(DocList *d, sqlite_int64 iDocid, DocList *pUpdate){
+ int modified = 0;
+ DocListReader reader;
+ char *p;
+
+ if( pUpdate!=NULL ){
+ assert( d->iType==pUpdate->iType);
+ assert( iDocid==firstDocid(pUpdate) );
+ }
+
+ readerInit(&reader, d);
+ while( !readerAtEnd(&reader) && peekDocid(&reader)<iDocid ){
+ skipDocument(&reader);
+ }
+
+ p = reader.p;
+ /* Delete if there is a matching element. */
+ if( !readerAtEnd(&reader) && iDocid==peekDocid(&reader) ){
+ skipDocument(&reader);
+ memmove(p, reader.p, docListEnd(d) - reader.p);
+ d->nData -= (reader.p - p);
+ modified = 1;
+ }
+
+ /* Insert if indicated. */
+ if( pUpdate!=NULL ){
+ int iDoclist = p-d->pData;
+ docListAddEndPos(pUpdate);
+
+ d->pData = realloc(d->pData, d->nData+pUpdate->nData);
+ p = d->pData + iDoclist;
+
+ memmove(p+pUpdate->nData, p, docListEnd(d) - p);
+ memcpy(p, pUpdate->pData, pUpdate->nData);
+ d->nData += pUpdate->nData;
+ modified = 1;
+ }
+
+ return modified;
+}
+
+/* Split the second half of doclist d into a separate doclist d2. Returns 1
+ * if successful, or 0 if d contains a single document and hence can't be
+ * split. */
+static int docListSplit(DocList *d, DocList *d2){
+ const char *pSplitPoint = d->pData + d->nData / 2;
+ DocListReader reader;
+
+ readerInit(&reader, d);
+ while( reader.p<pSplitPoint ){
+ skipDocument(&reader);
+ }
+ if( readerAtEnd(&reader) ) return 0;
+ docListInit(d2, d->iType, reader.p, docListEnd(d) - reader.p);
+ d->nData = reader.p - d->pData;
+ d->pData = realloc(d->pData, d->nData);
+ return 1;
+}
+
+/* A DocListMerge computes the AND of an in-memory DocList [in] and a chunked
+ * on-disk doclist, resulting in another in-memory DocList [out]. [in]
+ * and [out] may or may not store position information according to the
+ * caller's wishes. The on-disk doclist always comes with positions.
+ *
+ * The caller must read each chunk of the on-disk doclist in succession and
+ * pass it to mergeBlock().
+ *
+ * If [in] has positions, then the merge output contains only documents with
+ * matching positions in the two input doclists. If [in] does not have
+ * positions, then the merge output contains all documents common to the two
+ * input doclists.
+ *
+ * If [in] is NULL, then the on-disk doclist is copied to [out] directly.
+ *
+ * A merge is performed using an integer [iOffset] provided by the caller.
+ * [iOffset] is subtracted from each position in the on-disk doclist for the
+ * purpose of position comparison; this is helpful in implementing phrase
+ * searches.
+ *
+ * A DocListMerge is not yet able to propagate offsets through query
+ * processing; we should add that capability soon.
+*/
+typedef struct DocListMerge {
+ DocListReader in;
+ DocList *pOut;
+ int iOffset;
+} DocListMerge;
+
+static void mergeInit(DocListMerge *m,
+ DocList *pIn, int iOffset, DocList *pOut){
+ readerInit(&m->in, pIn);
+ m->pOut = pOut;
+ m->iOffset = iOffset;
+
+ /* can't handle offsets yet */
+ assert( pIn==NULL || pIn->iType <= DL_POSITIONS );
+ assert( pOut->iType <= DL_POSITIONS );
+}
+
+/* A helper function for mergeBlock(), below. Merge the position lists
+ * pointed to by m->in and pBlockReader.
+ * If the merge matches, write [iDocid] to m->pOut; if m->pOut
+ * has positions then write all matching positions as well. */
+static void mergePosList(DocListMerge *m, sqlite_int64 iDocid,
+ DocListReader *pBlockReader){
+ int block_pos = readPosition(pBlockReader);
+ int in_pos = readPosition(&m->in);
+ int match = 0;
+ while( block_pos!=-1 || in_pos!=-1 ){
+ if( block_pos-m->iOffset==in_pos ){
+ if( !match ){
+ docListAddDocid(m->pOut, iDocid);
+ match = 1;
+ }
+ if( m->pOut->iType >= DL_POSITIONS ){
+ docListAddPos(m->pOut, in_pos);
+ }
+ block_pos = readPosition(pBlockReader);
+ in_pos = readPosition(&m->in);
+ } else if( in_pos==-1 || (block_pos!=-1 && block_pos-m->iOffset<in_pos) ){
+ block_pos = readPosition(pBlockReader);
+ } else {
+ in_pos = readPosition(&m->in);
+ }
+ }
+ if( m->pOut->iType >= DL_POSITIONS && match ){
+ docListAddEndPos(m->pOut);
+ }
+}
+
+/* Merge one block of an on-disk doclist into a DocListMerge. */
+static void mergeBlock(DocListMerge *m, DocList *pBlock){
+ DocListReader blockReader;
+ assert( pBlock->iType >= DL_POSITIONS );
+ readerInit(&blockReader, pBlock);
+ while( !readerAtEnd(&blockReader) ){
+ sqlite_int64 iDocid = readDocid(&blockReader);
+ if( m->in.pDoclist!=NULL ){
+ while( 1 ){
+ if( readerAtEnd(&m->in) ) return; /* nothing more to merge */
+ if( peekDocid(&m->in)>=iDocid ) break;
+ skipDocument(&m->in);
+ }
+ if( peekDocid(&m->in)>iDocid ){ /* [pIn] has no match with iDocid */
+ skipPositionList(&blockReader); /* skip this docid in the block */
+ continue;
+ }
+ readDocid(&m->in);
+ }
+ /* We have a document match. */
+ if( m->in.pDoclist==NULL || m->in.pDoclist->iType < DL_POSITIONS ){
+ /* We don't need to do a poslist merge. */
+ docListAddDocid(m->pOut, iDocid);
+ if( m->pOut->iType >= DL_POSITIONS ){
+ /* Copy all positions to the output doclist. */
+ while( 1 ){
+ int pos = readPosition(&blockReader);
+ if( pos==-1 ) break;
+ docListAddPos(m->pOut, pos);
+ }
+ docListAddEndPos(m->pOut);
+ } else skipPositionList(&blockReader);
+ continue;
+ }
+ mergePosList(m, iDocid, &blockReader);
+ }
+}
+
+static char *string_dup_n(const char *s, int n){
+ char *str = malloc(n + 1);
+ memcpy(str, s, n);
+ str[n] = '\0';
+ return str;
+}
+
+/* Duplicate a string; the caller must free() the returned string.
+ * (We don't use strdup() since it's not part of the standard C library and
+ * may not be available everywhere.) */
+static char *string_dup(const char *s){
+ return string_dup_n(s, strlen(s));
+}
+
+/* Format a string, replacing each occurrence of the % character with
+ * zName. This may be more convenient than sqlite_mprintf()
+ * when one string is used repeatedly in a format string.
+ * The caller must free() the returned string. */
+static char *string_format(const char *zFormat, const char *zName){
+ const char *p;
+ size_t len = 0;
+ size_t nName = strlen(zName);
+ char *result;
+ char *r;
+
+ /* first compute length needed */
+ for(p = zFormat ; *p ; ++p){
+ len += (*p=='%' ? nName : 1);
+ }
+ len += 1; /* for null terminator */
+
+ r = result = malloc(len);
+ for(p = zFormat; *p; ++p){
+ if( *p=='%' ){
+ memcpy(r, zName, nName);
+ r += nName;
+ } else {
+ *r++ = *p;
+ }
+ }
+ *r++ = '\0';
+ assert( r == result + len );
+ return result;
+}
+
+static int sql_exec(sqlite3 *db, const char *zName, const char *zFormat){
+ char *zCommand = string_format(zFormat, zName);
+ int rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
+ free(zCommand);
+ return rc;
+}
+
+static int sql_prepare(sqlite3 *db, const char *zName, sqlite3_stmt **ppStmt,
+ const char *zFormat){
+ char *zCommand = string_format(zFormat, zName);
+ int rc = sqlite3_prepare(db, zCommand, -1, ppStmt, NULL);
+ free(zCommand);
+ return rc;
+}
+
+/* end utility functions */
+
+#define QUERY_GENERIC 0
+#define QUERY_FULLTEXT 1
+
+#define CHUNK_MAX 1024
+
+typedef enum fulltext_statement {
+ CONTENT_INSERT_STMT,
+ CONTENT_SELECT_STMT,
+ CONTENT_DELETE_STMT,
+
+ TERM_SELECT_STMT,
+ TERM_CHUNK_SELECT_STMT,
+ TERM_INSERT_STMT,
+ TERM_UPDATE_STMT,
+ TERM_DELETE_STMT,
+
+ MAX_STMT /* Always at end! */
+} fulltext_statement;
+
+/* These must exactly match the enum above. */
+/* TODO(adam): Is there some risk that a statement (in particular,
+** pTermSelectStmt) will be used in two cursors at once, e.g. if a
+** query joins a virtual table to itself? If so perhaps we should
+** move some of these to the cursor object.
+*/
+static const char *fulltext_zStatement[MAX_STMT] = {
+ /* CONTENT_INSERT */ "insert into %_content (rowid, content) values (?, ?)",
+ /* CONTENT_SELECT */ "select content from %_content where rowid = ?",
+ /* CONTENT_DELETE */ "delete from %_content where rowid = ?",
+
+ /* TERM_SELECT */
+ "select rowid, doclist from %_term where term = ? and first = ?",
+ /* TERM_CHUNK_SELECT */
+ "select max(first) from %_term where term = ? and first <= ?",
+ /* TERM_INSERT */
+ "insert into %_term (term, first, doclist) values (?, ?, ?)",
+ /* TERM_UPDATE */ "update %_term set doclist = ? where rowid = ?",
+ /* TERM_DELETE */ "delete from %_term where rowid = ?",
+};
+
+typedef struct fulltext_vtab {
+ sqlite3_vtab base;
+ sqlite3 *db;
+ const char *zName; /* virtual table name */
+ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
+
+ /* Precompiled statements which we keep as long as the table is
+ ** open.
+ */
+ sqlite3_stmt *pFulltextStatements[MAX_STMT];
+} fulltext_vtab;
+
+typedef struct fulltext_cursor {
+ sqlite3_vtab_cursor base;
+ int iCursorType; /* QUERY_GENERIC or QUERY_FULLTEXT */
+
+ sqlite3_stmt *pStmt;
+
+ int eof;
+
+ /* The following is used only when iCursorType == QUERY_FULLTEXT. */
+ DocListReader result;
+} fulltext_cursor;
+
+static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){
+ return (fulltext_vtab *) c->base.pVtab;
+}
+
+static sqlite3_module fulltextModule; /* forward declaration */
+
+/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
+** If the indicated statement has never been prepared, it is prepared
+** and cached, otherwise the cached version is reset.
+*/
+static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
+ sqlite3_stmt **ppStmt){
+ assert( iStmt<MAX_STMT );
+ if( v->pFulltextStatements[iStmt]==NULL ){
+ int rc = sql_prepare(v->db, v->zName, &v->pFulltextStatements[iStmt],
+ fulltext_zStatement[iStmt]);
+ if( rc!=SQLITE_OK ) return rc;
+ } else {
+ int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+
+ *ppStmt = v->pFulltextStatements[iStmt];
+ return SQLITE_OK;
+}
+
+/* Step the indicated statement, handling errors SQLITE_BUSY (by
+** retrying) and SQLITE_SCHEMA (by re-preparing and transferring
+** bindings to the new statement).
+** TODO(adam): We should extend this function so that it can work with
+** statements declared locally, not only globally cached statements.
+*/
+static int sql_step_statement(fulltext_vtab *v, fulltext_statement iStmt,
+ sqlite3_stmt **ppStmt){
+ int rc;
+ sqlite3_stmt *s = *ppStmt;
+ assert( iStmt<MAX_STMT );
+ assert( s==v->pFulltextStatements[iStmt] );
+
+ while( (rc=sqlite3_step(s))!=SQLITE_DONE && rc!=SQLITE_ROW ){
+ sqlite3_stmt *pNewStmt;
+
+ if( rc==SQLITE_BUSY ) continue;
+ if( rc!=SQLITE_ERROR ) return rc;
+
+ rc = sqlite3_reset(s);
+ if( rc!=SQLITE_SCHEMA ) return SQLITE_ERROR;
+
+ v->pFulltextStatements[iStmt] = NULL; /* Still in s */
+ rc = sql_get_statement(v, iStmt, &pNewStmt);
+ if( rc!=SQLITE_OK ) goto err;
+ *ppStmt = pNewStmt;
+
+ rc = sqlite3_transfer_bindings(s, pNewStmt);
+ if( rc!=SQLITE_OK ) goto err;
+
+ rc = sqlite3_finalize(s);
+ if( rc!=SQLITE_OK ) return rc;
+ s = pNewStmt;
+ }
+ return rc;
+
+ err:
+ sqlite3_finalize(s);
+ return rc;
+}
+
+/* Like sql_step_statement(), but convert SQLITE_DONE to SQLITE_OK.
+** Useful for statements like UPDATE, where we expect no results.
+*/
+static int sql_single_step_statement(fulltext_vtab *v,
+ fulltext_statement iStmt,
+ sqlite3_stmt **ppStmt){
+ int rc = sql_step_statement(v, iStmt, ppStmt);
+ return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
+}
+
+/* insert into %_content (rowid, content) values ([rowid], [zContent]) */
+static int content_insert(fulltext_vtab *v, sqlite3_value *rowid,
+ const char *zContent, int nContent){
+ sqlite3_stmt *s;
+ int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_value(s, 1, rowid);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_text(s, 2, zContent, nContent, SQLITE_STATIC);
+ if( rc!=SQLITE_OK ) return rc;
+
+ return sql_single_step_statement(v, CONTENT_INSERT_STMT, &s);
+}
+
+/* select content from %_content where rowid = [iRow]
+ * The caller must delete the returned string. */
+static int content_select(fulltext_vtab *v, sqlite_int64 iRow,
+ char **pzContent){
+ sqlite3_stmt *s;
+ int rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_int64(s, 1, iRow);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sql_step_statement(v, CONTENT_SELECT_STMT, &s);
+ if( rc!=SQLITE_ROW ) return rc;
+
+ *pzContent = string_dup((const char *)sqlite3_column_text(s, 0));
+
+ /* We expect only one row. We must execute another sqlite3_step()
+ * to complete the iteration; otherwise the table will remain locked. */
+ rc = sqlite3_step(s);
+ if( rc==SQLITE_DONE ) return SQLITE_OK;
+
+ free(*pzContent);
+ return rc;
+}
+
+/* delete from %_content where rowid = [iRow ] */
+static int content_delete(fulltext_vtab *v, sqlite_int64 iRow){
+ sqlite3_stmt *s;
+ int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_int64(s, 1, iRow);
+ if( rc!=SQLITE_OK ) return rc;
+
+ return sql_single_step_statement(v, CONTENT_DELETE_STMT, &s);
+}
+
+/* select rowid, doclist from %_term where term = [zTerm] and first = [iFirst]
+ * If found, returns SQLITE_OK; the caller must free the returned doclist.
+ * If no rows found, returns SQLITE_ERROR. */
+static int term_select(fulltext_vtab *v, const char *zTerm, int nTerm,
+ sqlite_int64 iFirst,
+ sqlite_int64 *rowid,
+ DocList *out){
+ sqlite3_stmt *s;
+ int rc = sql_get_statement(v, TERM_SELECT_STMT, &s);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_text(s, 1, zTerm, nTerm, SQLITE_TRANSIENT);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_int64(s, 2, iFirst);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sql_step_statement(v, TERM_SELECT_STMT, &s);
+ if( rc!=SQLITE_ROW ) return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
+
+ *rowid = sqlite3_column_int64(s, 0);
+ docListInit(out, DL_POSITIONS_OFFSETS,
+ sqlite3_column_blob(s, 1), sqlite3_column_bytes(s, 1));
+
+ /* We expect only one row. We must execute another sqlite3_step()
+ * to complete the iteration; otherwise the table will remain locked. */
+ rc = sqlite3_step(s);
+ return rc==SQLITE_DONE ? SQLITE_OK : rc;
+}
+
+/* select max(first) from %_term where term = [zTerm] and first <= [iFirst]
+ * If found, returns SQLITE_ROW and result in *piResult; if the query returns
+ * NULL (meaning no row found) returns SQLITE_DONE.
+ */
+static int term_chunk_select(fulltext_vtab *v, const char *zTerm, int nTerm,
+ sqlite_int64 iFirst, sqlite_int64 *piResult){
+ sqlite3_stmt *s;
+ int rc = sql_get_statement(v, TERM_CHUNK_SELECT_STMT, &s);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_text(s, 1, zTerm, nTerm, SQLITE_STATIC);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_int64(s, 2, iFirst);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sql_step_statement(v, TERM_CHUNK_SELECT_STMT, &s);
+ if( rc!=SQLITE_ROW ) return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
+
+ switch( sqlite3_column_type(s, 0) ){
+ case SQLITE_NULL:
+ rc = SQLITE_DONE;
+ break;
+ case SQLITE_INTEGER:
+ *piResult = sqlite3_column_int64(s, 0);
+ break;
+ default:
+ return SQLITE_ERROR;
+ }
+ /* We expect only one row. We must execute another sqlite3_step()
+ * to complete the iteration; otherwise the table will remain locked. */
+ if( sqlite3_step(s) != SQLITE_DONE ) return SQLITE_ERROR;
+ return rc;
+}
+
+/* insert into %_term (term, first, doclist)
+ values ([zTerm], [iFirst], [doclist]) */
+static int term_insert(fulltext_vtab *v, const char *zTerm, int nTerm,
+ sqlite_int64 iFirst, DocList *doclist){
+ sqlite3_stmt *s;
+ int rc = sql_get_statement(v, TERM_INSERT_STMT, &s);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_text(s, 1, zTerm, nTerm, SQLITE_STATIC);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_int64(s, 2, iFirst);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_blob(s, 3, doclist->pData, doclist->nData, SQLITE_STATIC);
+ if( rc!=SQLITE_OK ) return rc;
+
+ return sql_single_step_statement(v, TERM_INSERT_STMT, &s);
+}
+
+/* update %_term set doclist = [doclist] where rowid = [rowid] */
+static int term_update(fulltext_vtab *v, sqlite_int64 rowid,
+ DocList *doclist){
+ sqlite3_stmt *s;
+ int rc = sql_get_statement(v, TERM_UPDATE_STMT, &s);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_blob(s, 1, doclist->pData, doclist->nData,
+ SQLITE_STATIC);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_int64(s, 2, rowid);
+ if( rc!=SQLITE_OK ) return rc;
+
+ return sql_single_step_statement(v, TERM_UPDATE_STMT, &s);
+}
+
+static int term_delete(fulltext_vtab *v, sqlite_int64 rowid){
+ sqlite3_stmt *s;
+ int rc = sql_get_statement(v, TERM_DELETE_STMT, &s);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_int64(s, 1, rowid);
+ if( rc!=SQLITE_OK ) return rc;
+
+ return sql_single_step_statement(v, TERM_DELETE_STMT, &s);
+}
+
+static void fulltext_vtab_destroy(fulltext_vtab *v){
+ int iStmt;
+
+ for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
+ if( v->pFulltextStatements[iStmt]!=NULL ){
+ sqlite3_finalize(v->pFulltextStatements[iStmt]);
+ v->pFulltextStatements[iStmt] = NULL;
+ }
+ }
+
+ if( v->pTokenizer!=NULL ){
+ v->pTokenizer->pModule->xDestroy(v->pTokenizer);
+ v->pTokenizer = NULL;
+ }
+
+ free((void *) v->zName);
+ free(v);
+}
+
+/* Current interface:
+** argv[0] - module name
+** argv[1] - database name
+** argv[2] - table name
+** argv[3] - tokenizer name (optional, a sensible default is provided)
+** argv[4..] - passed to tokenizer (optional based on tokenizer)
+**/
+static int fulltextConnect(sqlite3 *db, void *pAux, int argc, char **argv,
+ sqlite3_vtab **ppVTab){
+ int rc;
+ fulltext_vtab *v;
+ sqlite3_tokenizer_module *m = NULL;
+
+ assert( argc>=3 );
+ v = (fulltext_vtab *) malloc(sizeof(fulltext_vtab));
+ /* sqlite will initialize v->base */
+ v->db = db;
+ v->zName = string_dup(argv[2]);
+ v->pTokenizer = NULL;
+
+ if( argc==3 ){
+ get_simple_tokenizer_module(&m);
+ } else {
+ /* TODO(shess) For now, add new tokenizers as else if clauses. */
+ if( !strcmp(argv[3], "simple") ){
+ get_simple_tokenizer_module(&m);
+ } else {
+ assert( "unrecognized tokenizer"==NULL );
+ }
+ }
+
+ /* TODO(shess) Since tokenization impacts the index, the parameters
+ ** to the tokenizer need to be identical when a persistent virtual
+ ** table is re-created. One solution would be a meta-table to track
+ ** such information in the database. Then we could verify that the
+ ** information is identical on subsequent creates.
+ */
+ /* TODO(shess) Why isn't argv already (const char **)? */
+ rc = m->xCreate(argc-3, (const char **) (argv+3), &v->pTokenizer);
+ if( rc!=SQLITE_OK ) return rc;
+ v->pTokenizer->pModule = m;
+
+ /* TODO: verify the existence of backing tables foo_content, foo_term */
+
+ rc = sqlite3_declare_vtab(db, "create table x(content text)");
+ if( rc!=SQLITE_OK ) return rc;
+
+ memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
+
+ *ppVTab = &v->base;
+ return SQLITE_OK;
+}
+
+static int fulltextCreate(sqlite3 *db, void *pAux, int argc, char **argv,
+ sqlite3_vtab **ppVTab){
+ int rc;
+ assert( argc>=3 );
+
+ /* The %_content table holds the text of each full-text item, with
+ ** the rowid used as the docid.
+ **
+ ** The %_term table maps each term to a document list blob
+ ** containing elements sorted by ascending docid, each element
+ ** encoded as:
+ **
+ ** docid varint-encoded
+ ** token count varint-encoded
+ ** "count" token elements (poslist):
+ ** position varint-encoded as delta from previous position
+ ** start offset varint-encoded as delta from previous start offset
+ ** end offset varint-encoded as delta from start offset
+ **
+ ** Additionally, doclist blobs can be chunked into multiple rows,
+ ** using "first" to order the blobs. "first" is simply the first
+ ** docid in the blob.
+ */
+ /*
+ ** NOTE(shess) That last sentence is incorrect in the face of
+ ** deletion, which can leave a doclist that doesn't contain the
+ ** first from that row. I _believe_ this does not matter to the
+ ** operation of the system, but it might be reasonable to update
+ ** appropriately in case this assumption becomes more important.
+ */
+ rc = sql_exec(db, argv[2],
+ "create table %_content(content text);"
+ "create table %_term(term text, first integer, doclist blob);"
+ "create index %_index on %_term(term, first)");
+ if( rc!=SQLITE_OK ) return rc;
+
+ return fulltextConnect(db, pAux, argc, argv, ppVTab);
+}
+
+/* Decide how to handle an SQL query.
+ * At the moment, MATCH queries can include implicit boolean ANDs; we
+ * haven't implemented phrase searches or OR yet. */
+static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
+ int i;
+
+ for(i=0; i<pInfo->nConstraint; ++i){
+ const struct sqlite3_index_constraint *pConstraint;
+ pConstraint = &pInfo->aConstraint[i];
+ if( pConstraint->iColumn==0 &&
+ pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH &&
+ pConstraint->usable ){ /* a full-text search */
+ pInfo->aConstraintUsage[i].argvIndex = 1;
+ pInfo->aConstraintUsage[i].omit = 1;
+ pInfo->idxNum = QUERY_FULLTEXT;
+ pInfo->estimatedCost = 1.0; /* an arbitrary value for now */
+ return SQLITE_OK;
+ }
+ }
+ pInfo->idxNum = QUERY_GENERIC;
+ return SQLITE_OK;
+}
+
+static int fulltextDisconnect(sqlite3_vtab *pVTab){
+ fulltext_vtab_destroy((fulltext_vtab *)pVTab);
+ return SQLITE_OK;
+}
+
+static int fulltextDestroy(sqlite3_vtab *pVTab){
+ fulltext_vtab *v = (fulltext_vtab *)pVTab;
+
+ int rc = sql_exec(v->db, v->zName,
+ "drop table %_content; drop table %_term");
+ if( rc!=SQLITE_OK ) return rc;
+
+ fulltext_vtab_destroy((fulltext_vtab *)pVTab);
+ return SQLITE_OK;
+}
+
+static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+ fulltext_cursor *c;
+
+ c = (fulltext_cursor *) calloc(sizeof(fulltext_cursor), 1);
+ /* sqlite will initialize c->base */
+ *ppCursor = &c->base;
+
+ return SQLITE_OK;
+}
+
+static int fulltextClose(sqlite3_vtab_cursor *pCursor){
+ fulltext_cursor *c = (fulltext_cursor *) pCursor;
+ sqlite3_finalize(c->pStmt);
+ if( c->result.pDoclist!=NULL ){
+ docListDelete(c->result.pDoclist);
+ }
+ free(c);
+ return SQLITE_OK;
+}
+
+static int fulltextNext(sqlite3_vtab_cursor *pCursor){
+ fulltext_cursor *c = (fulltext_cursor *) pCursor;
+ sqlite_int64 iDocid;
+ int rc;
+
+ switch( c->iCursorType ){
+ case QUERY_GENERIC:
+ /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
+ rc = sqlite3_step(c->pStmt);
+ switch( rc ){
+ case SQLITE_ROW:
+ c->eof = 0;
+ return SQLITE_OK;
+ case SQLITE_DONE:
+ c->eof = 1;
+ return SQLITE_OK;
+ default:
+ c->eof = 1;
+ return rc;
+ }
+ case QUERY_FULLTEXT:
+ rc = sqlite3_reset(c->pStmt);
+ if( rc!=SQLITE_OK ) return rc;
+
+ if( readerAtEnd(&c->result)){
+ c->eof = 1;
+ return SQLITE_OK;
+ }
+ iDocid = readDocid(&c->result);
+ rc = sqlite3_bind_int64(c->pStmt, 1, iDocid);
+ if( rc!=SQLITE_OK ) return rc;
+ /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
+ rc = sqlite3_step(c->pStmt);
+ if( rc==SQLITE_ROW ){ /* the case we expect */
+ c->eof = 0;
+ return SQLITE_OK;
+ }
+ /* an error occurred; abort */
+ return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
+ default:
+ assert( 0 );
+ return SQLITE_ERROR; /* not reached */
+ }
+}
+
+static int term_select_doclist(fulltext_vtab *v, const char *pTerm, int nTerm,
+ sqlite3_stmt **ppStmt){
+ int rc;
+ if( *ppStmt ){
+ rc = sqlite3_reset(*ppStmt);
+ } else {
+ rc = sql_prepare(v->db, v->zName, ppStmt,
+ "select doclist from %_term where term = ? order by first");
+ }
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_bind_text(*ppStmt, 1, pTerm, nTerm, SQLITE_TRANSIENT);
+ if( rc!=SQLITE_OK ) return rc;
+
+ return sqlite3_step(*ppStmt); /* TODO(adamd): handle schema error */
+}
+
+/* Read the posting list for [zTerm]; AND it with the doclist [in] to
+ * produce the doclist [out], using the given offset [iOffset] for phrase
+ * matching.
+ * (*pSelect) is used to hold an SQLite statement used inside this function;
+ * the caller should initialize *pSelect to NULL before the first call.
+ */
+static int query_merge(fulltext_vtab *v, sqlite3_stmt **pSelect,
+ const char *zTerm,
+ DocList *pIn, int iOffset, DocList *out){
+ int rc;
+ DocListMerge merge;
+
+ if( pIn!=NULL && !pIn->nData ){
+ /* If [pIn] is already empty, there's no point in reading the
+ * posting list to AND it in; return immediately. */
+ return SQLITE_OK;
+ }
+
+ rc = term_select_doclist(v, zTerm, -1, pSelect);
+ if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
+
+ mergeInit(&merge, pIn, iOffset, out);
+ while( rc==SQLITE_ROW ){
+ DocList block;
+ docListInit(&block, DL_POSITIONS_OFFSETS,
+ sqlite3_column_blob(*pSelect, 0),
+ sqlite3_column_bytes(*pSelect, 0));
+ mergeBlock(&merge, &block);
+ docListDestroy(&block);
+
+ rc = sqlite3_step(*pSelect);
+ if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
+ return rc;
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+typedef struct QueryTerm {
+ int is_phrase; /* true if this term begins a new phrase */
+ const char *zTerm;
+} QueryTerm;
+
+/* A parsed query.
+ *
+ * As an example, parsing the query ["four score" years "new nation"] will
+ * yield a Query with 5 terms:
+ * "four", is_phrase = 1
+ * "score", is_phrase = 0
+ * "years", is_phrase = 1
+ * "new", is_phrase = 1
+ * "nation", is_phrase = 0
+ */
+typedef struct Query {
+ int nTerms;
+ QueryTerm *pTerm;
+} Query;
+
+static void query_add(Query *q, int is_phrase, const char *zTerm){
+ QueryTerm *t;
+ ++q->nTerms;
+ q->pTerm = realloc(q->pTerm, q->nTerms * sizeof(q->pTerm[0]));
+ t = &q->pTerm[q->nTerms - 1];
+ t->is_phrase = is_phrase;
+ t->zTerm = zTerm;
+}
+
+static void query_free(Query *q){
+ int i;
+ for(i = 0; i < q->nTerms; ++i){
+ free((void *) q->pTerm[i].zTerm);
+ }
+ free(q->pTerm);
+}
+
+static int tokenize_segment(sqlite3_tokenizer *pTokenizer,
+ const char *zQuery, int in_phrase,
+ Query *pQuery){
+ sqlite3_tokenizer_module *pModule = pTokenizer->pModule;
+ sqlite3_tokenizer_cursor *pCursor;
+ int is_first = 1;
+
+ int rc = pModule->xOpen(pTokenizer, zQuery, -1, &pCursor);
+ if( rc!=SQLITE_OK ) return rc;
+ pCursor->pTokenizer = pTokenizer;
+
+ while( 1 ){
+ const char *zToken;
+ int nToken, iStartOffset, iEndOffset, dummy_pos;
+
+ rc = pModule->xNext(pCursor,
+ &zToken, &nToken,
+ &iStartOffset, &iEndOffset,
+ &dummy_pos);
+ if( rc!=SQLITE_OK ) break;
+ query_add(pQuery, !in_phrase || is_first, string_dup_n(zToken, nToken));
+ is_first = 0;
+ }
+
+ return pModule->xClose(pCursor);
+}
+
+/* Parse a query string, yielding a Query object. */
+static int parse_query(fulltext_vtab *v, const char *zQuery, Query *pQuery){
+ char *zQuery1 = string_dup(zQuery);
+ int in_phrase = 0;
+ char *s = zQuery1;
+ pQuery->nTerms = 0;
+ pQuery->pTerm = NULL;
+
+ while( *s ){
+ char *t = s;
+ while( *t ){
+ if( *t=='"' ){
+ *t++ = '\0';
+ break;
+ }
+ ++t;
+ }
+ if( *s ){
+ tokenize_segment(v->pTokenizer, s, in_phrase, pQuery);
+ }
+ s = t;
+ in_phrase = !in_phrase;
+ }
+
+ free(zQuery1);
+ return SQLITE_OK;
+}
+
+/* Perform a full-text query; return a list of documents in [pResult]. */
+static int fulltext_query(fulltext_vtab *v, const char *zQuery,
+ DocList **pResult){
+ Query q;
+ int phrase_start = -1;
+ int i;
+ sqlite3_stmt *pSelect = NULL;
+ DocList *d = NULL;
+
+ int rc = parse_query(v, zQuery, &q);
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* Merge terms. */
+ for(i = 0 ; i < q.nTerms ; ++i){
+ /* In each merge step, we need to generate positions whenever we're
+ * processing a phrase which hasn't ended yet. */
+ int need_positions = i<q.nTerms-1 && !q.pTerm[i+1].is_phrase;
+ DocList *next = docListNew(need_positions ? DL_POSITIONS : DL_DOCIDS);
+ if( q.pTerm[i].is_phrase ){
+ phrase_start = i;
+ }
+ rc = query_merge(v, &pSelect, q.pTerm[i].zTerm, d, i - phrase_start, next);
+ if( rc!=SQLITE_OK ) break;
+ if( d!=NULL ){
+ docListDelete(d);
+ }
+ d = next;
+ }
+
+ sqlite3_finalize(pSelect);
+ query_free(&q);
+ *pResult = d;
+ return rc;
+}
+
+static int fulltextFilter(sqlite3_vtab_cursor *pCursor,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv){
+ fulltext_cursor *c = (fulltext_cursor *) pCursor;
+ fulltext_vtab *v = cursor_vtab(c);
+ int rc;
+ const char *zStatement;
+
+ c->iCursorType = idxNum;
+ switch( idxNum ){
+ case QUERY_GENERIC:
+ zStatement = "select rowid, content from %_content";
+ break;
+
+ case QUERY_FULLTEXT: /* full-text search */
+ {
+ const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
+ DocList *pResult;
+ assert( argc==1 );
+ rc = fulltext_query(v, zQuery, &pResult);
+ if( rc!=SQLITE_OK ) return rc;
+ readerInit(&c->result, pResult);
+ zStatement = "select rowid, content from %_content where rowid = ?";
+ break;
+ }
+
+ default:
+ assert( 0 );
+ }
+
+ rc = sql_prepare(v->db, v->zName, &c->pStmt, zStatement);
+ if( rc!=SQLITE_OK ) return rc;
+
+ return fulltextNext(pCursor);
+}
+
+static int fulltextEof(sqlite3_vtab_cursor *pCursor){
+ fulltext_cursor *c = (fulltext_cursor *) pCursor;
+ return c->eof;
+}
+
+static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
+ sqlite3_context *pContext, int idxCol){
+ fulltext_cursor *c = (fulltext_cursor *) pCursor;
+ const char *s;
+
+ assert( idxCol==0 );
+ s = (const char *) sqlite3_column_text(c->pStmt, 1);
+ sqlite3_result_text(pContext, s, -1, SQLITE_TRANSIENT);
+
+ return SQLITE_OK;
+}
+
+static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+ fulltext_cursor *c = (fulltext_cursor *) pCursor;
+
+ *pRowid = sqlite3_column_int64(c->pStmt, 0);
+ return SQLITE_OK;
+}
+
+/* Build a hash table containing all terms in zText. */
+static int build_terms(Hash *terms, sqlite3_tokenizer *pTokenizer,
+ const char *zText, sqlite_int64 iDocid){
+ sqlite3_tokenizer_cursor *pCursor;
+ const char *pToken;
+ int nTokenBytes;
+ int iStartOffset, iEndOffset, iPosition;
+
+ int rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
+ if( rc!=SQLITE_OK ) return rc;
+
+ pCursor->pTokenizer = pTokenizer;
+ HashInit(terms, HASH_STRING, 1);
+ while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
+ &pToken, &nTokenBytes,
+ &iStartOffset, &iEndOffset,
+ &iPosition) ){
+ DocList *p;
+
+ /* Positions can't be negative; we use -1 as a terminator internally. */
+ if( iPosition<0 ) {
+ rc = SQLITE_ERROR;
+ goto err;
+ }
+
+ p = HashFind(terms, pToken, nTokenBytes);
+ if( p==NULL ){
+ p = docListNew(DL_POSITIONS_OFFSETS);
+ docListAddDocid(p, iDocid);
+ HashInsert(terms, pToken, nTokenBytes, p);
+ }
+ docListAddPosOffset(p, iPosition, iStartOffset, iEndOffset);
+ }
+
+err:
+ /* TODO(shess) Check return? Should this be able to cause errors at
+ ** this point? Actually, same question about sqlite3_finalize(),
+ ** though one could argue that failure there means that the data is
+ ** not durable. *ponder*
+ */
+ pTokenizer->pModule->xClose(pCursor);
+ return rc;
+}
+/* Update the %_terms table to map the term [zTerm] to the given rowid. */
+static int index_insert_term(fulltext_vtab *v, const char *zTerm, int nTerm,
+ sqlite_int64 iDocid, DocList *p){
+ sqlite_int64 iFirst;
+ sqlite_int64 iIndexRow;
+ DocList doclist;
+
+ int rc = term_chunk_select(v, zTerm, nTerm, iDocid, &iFirst);
+ if( rc==SQLITE_DONE ){
+ docListInit(&doclist, DL_POSITIONS_OFFSETS, 0, 0);
+ if( docListUpdate(&doclist, iDocid, p) ){
+ rc = term_insert(v, zTerm, nTerm, iDocid, &doclist);
+ docListDestroy(&doclist);
+ return rc;
+ }
+ return SQLITE_OK;
+ }
+ if( rc!=SQLITE_ROW ) return SQLITE_ERROR;
+
+ /* This word is in the index; add this document ID to its blob. */
+
+ rc = term_select(v, zTerm, nTerm, iFirst, &iIndexRow, &doclist);
+ if( rc!=SQLITE_OK ) return rc;
+
+ if( docListUpdate(&doclist, iDocid, p) ){
+ /* If the blob is too big, split it in half. */
+ if( doclist.nData>CHUNK_MAX ){
+ DocList half;
+ if( docListSplit(&doclist, &half) ){
+ rc = term_insert(v, zTerm, nTerm, firstDocid(&half), &half);
+ docListDestroy(&half);
+ if( rc!=SQLITE_OK ) goto err;
+ }
+ }
+ rc = term_update(v, iIndexRow, &doclist);
+ }
+
+err:
+ docListDestroy(&doclist);
+ return rc;
+}
+
+/* Insert a row into the full-text index; set *piRowid to be the ID of the
+ * new row. */
+static int index_insert(fulltext_vtab *v,
+ sqlite3_value *pRequestRowid, const char *zText,
+ sqlite_int64 *piRowid){
+ Hash terms; /* maps term string -> PosList */
+ HashElem *e;
+
+ int rc = content_insert(v, pRequestRowid, zText, -1);
+ if( rc!=SQLITE_OK ) return rc;
+ *piRowid = sqlite3_last_insert_rowid(v->db);
+
+ if( !zText ) return SQLITE_OK; /* nothing to index */
+
+ rc = build_terms(&terms, v->pTokenizer, zText, *piRowid);
+ if( rc!=SQLITE_OK ) return rc;
+
+ for(e=HashFirst(&terms); e; e=HashNext(e)){
+ DocList *p = HashData(e);
+ rc = index_insert_term(v, HashKey(e), HashKeysize(e), *piRowid, p);
+ if( rc!=SQLITE_OK ) break;
+ }
+
+ for(e=HashFirst(&terms); e; e=HashNext(e)){
+ DocList *p = HashData(e);
+ docListDelete(p);
+ }
+ HashClear(&terms);
+ return rc;
+}
+
+static int index_delete_term(fulltext_vtab *v, const char *zTerm, int nTerm,
+ sqlite_int64 iDocid){
+ sqlite_int64 iFirst;
+ sqlite_int64 iIndexRow;
+ DocList doclist;
+
+ int rc = term_chunk_select(v, zTerm, nTerm, iDocid, &iFirst);
+ if( rc!=SQLITE_ROW ) return SQLITE_ERROR;
+
+ rc = term_select(v, zTerm, nTerm, iFirst, &iIndexRow, &doclist);
+ if( rc!=SQLITE_OK ) return rc;
+
+ if( docListUpdate(&doclist, iDocid, NULL) ){
+ if( doclist.nData>0 ){
+ rc = term_update(v, iIndexRow, &doclist);
+ } else { /* empty posting list */
+ rc = term_delete(v, iIndexRow);
+ }
+ }
+ docListDestroy(&doclist);
+ return rc;
+}
+
+/* Delete a row from the full-text index. */
+static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
+ char *zText;
+ Hash terms;
+ HashElem *e;
+
+ int rc = content_select(v, iRow, &zText);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = build_terms(&terms, v->pTokenizer, zText, iRow);
+ free(zText);
+ if( rc!=SQLITE_OK ) return rc;
+
+ for(e=HashFirst(&terms); e; e=HashNext(e)){
+ rc = index_delete_term(v, HashKey(e), HashKeysize(e), iRow);
+ if( rc!=SQLITE_OK ) break;
+ }
+ for(e=HashFirst(&terms); e; e=HashNext(e)){
+ DocList *p = HashData(e);
+ docListDelete(p);
+ }
+ HashClear(&terms);
+
+ return content_delete(v, iRow);
+}
+
+static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
+ sqlite_int64 *pRowid){
+ fulltext_vtab *v = (fulltext_vtab *) pVtab;
+
+ if( nArg<2 ){
+ return index_delete(v, sqlite3_value_int64(ppArg[0]));
+ }
+
+ if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
+ return SQLITE_ERROR; /* an update; not yet supported */
+ }
+
+ assert( nArg==3 ); /* ppArg[1] = rowid, ppArg[2] = content */
+ return index_insert(v, ppArg[1],
+ (const char *)sqlite3_value_text(ppArg[2]), pRowid);
+}
+
+static sqlite3_module fulltextModule = {
+ 0,
+ fulltextCreate,
+ fulltextConnect,
+ fulltextBestIndex,
+ fulltextDisconnect,
+ fulltextDestroy,
+ fulltextOpen,
+ fulltextClose,
+ fulltextFilter,
+ fulltextNext,
+ fulltextEof,
+ fulltextColumn,
+ fulltextRowid,
+ fulltextUpdate
+};
+
+int fulltext_init(sqlite3 *db){
+ return sqlite3_create_module(db, "fulltext", &fulltextModule, 0);
+}
+
+#if !SQLITE_CORE
+int sqlite3_extension_init(sqlite3 *db, char **pzErrMsg,
+ const sqlite3_api_routines *pApi){
+ SQLITE_EXTENSION_INIT2(pApi)
+ return fulltext_init(db);
+}
+#endif
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-01 20:52:47 +0000