1 files changed, 36 insertions, 7 deletions

diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c
index 1c3a790..a6e3492 100644
--- a/ext/fts3/fts3_expr.c
+++ b/ext/fts3/fts3_expr.c
@@ -92,6 +92,7 @@ int sqlite3_fts3_enable_parentheses = 0;
 typedef struct ParseContext ParseContext;
 struct ParseContext {
   sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
+  int iLangid;                        /* Language id used with tokenizer */
   const char **azCol;                 /* Array of column names for fts3 table */
   int bFts4;                          /* True to allow FTS4-only syntax */
   int nCol;                           /* Number of entries in azCol[] */
@@ -127,6 +128,33 @@ static void *fts3MallocZero(int nByte){
   return pRet;
 }
 
+int sqlite3Fts3OpenTokenizer(
+  sqlite3_tokenizer *pTokenizer,
+  int iLangid,
+  const char *z,
+  int n,
+  sqlite3_tokenizer_cursor **ppCsr
+){
+  sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+  sqlite3_tokenizer_cursor *pCsr = 0;
+  int rc;
+
+  rc = pModule->xOpen(pTokenizer, z, n, &pCsr);
+  assert( rc==SQLITE_OK || pCsr==0 );
+  if( rc==SQLITE_OK ){
+    pCsr->pTokenizer = pTokenizer;
+    if( pModule->iVersion>=1 ){
+      rc = pModule->xLanguageid(pCsr, iLangid);
+      if( rc!=SQLITE_OK ){
+        pModule->xClose(pCsr);
+        pCsr = 0;
+      }
+    }
+  }
+  *ppCsr = pCsr;
+  return rc;
+}
+
 
 /*
 ** Extract the next token from buffer z (length n) using the tokenizer
@@ -154,15 +182,13 @@ static int getNextToken(
   Fts3Expr *pRet = 0;
   int nConsumed = 0;
 
-  rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
+  rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor);
   if( rc==SQLITE_OK ){
     const char *zToken;
     int nToken, iStart, iEnd, iPosition;
     int nByte;                               /* total space to allocate */
 
-    pCursor->pTokenizer = pTokenizer;
     rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
-
     if( rc==SQLITE_OK ){
       nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
       pRet = (Fts3Expr *)fts3MallocZero(nByte);
@@ -268,10 +294,10 @@ static int getNextString(
   ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
   ** structures.
   */
-  rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
+  rc = sqlite3Fts3OpenTokenizer(
+      pTokenizer, pParse->iLangid, zInput, nInput, &pCursor);
   if( rc==SQLITE_OK ){
     int ii;
-    pCursor->pTokenizer = pTokenizer;
     for(ii=0; rc==SQLITE_OK; ii++){
       const char *zByte;
       int nByte, iBegin, iEnd, iPos;
@@ -745,6 +771,7 @@ exprparse_out:
 */
 int sqlite3Fts3ExprParse(
   sqlite3_tokenizer *pTokenizer,      /* Tokenizer module */
+  int iLangid,                        /* Language id for tokenizer */
   char **azCol,                       /* Array of column names for fts3 table */
   int bFts4,                          /* True to allow FTS4-only syntax */
   int nCol,                           /* Number of entries in azCol[] */
@@ -755,11 +782,13 @@ int sqlite3Fts3ExprParse(
   int nParsed;
   int rc;
   ParseContext sParse;
+
+  memset(&sParse, 0, sizeof(ParseContext));
   sParse.pTokenizer = pTokenizer;
+  sParse.iLangid = iLangid;
   sParse.azCol = (const char **)azCol;
   sParse.nCol = nCol;
   sParse.iDefaultCol = iDefaultCol;
-  sParse.nNest = 0;
   sParse.bFts4 = bFts4;
   if( z==0 ){
     *ppExpr = 0;
@@ -950,7 +979,7 @@ static void fts3ExprTest(
   }
 
   rc = sqlite3Fts3ExprParse(
-      pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
+      pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
   );
   if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
     sqlite3_result_error(context, "Error parsing expression", -1);