Imported Upstream version 2.0.3

1 files changed, 463 insertions, 0 deletions

diff --git a/test/fts3snippet.test b/test/fts3snippet.test
new file mode 100644
index 0000000..b8646cd
--- /dev/null
+++ b/test/fts3snippet.test
@@ -0,0 +1,463 @@
+# 2010 January 07
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# The tests in this file test the FTS3 auxillary functions offsets(), 
+# snippet() and matchinfo() work. At time of writing, running this file 
+# provides full coverage of fts3_snippet.c.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
+ifcapable !fts3 { finish_test ; return }
+source $testdir/fts3_common.tcl
+
+set sqlite_fts3_enable_parentheses 1
+set DO_MALLOC_TEST 0
+
+# Transform the list $L to its "normal" form. So that it can be compared to
+# another list with the same set of elements using [string compare].
+#
+proc normalize {L} {
+  set ret [list]
+  foreach l $L {lappend ret $l}
+  return $ret
+}
+
+proc do_offsets_test {name expr args} {
+  set result [list]
+  foreach a $args {
+    lappend result [normalize $a]
+  }
+  do_select_test $name {
+    SELECT offsets(ft) FROM ft WHERE ft MATCH $expr
+  } $result
+}
+  
+# Document text used by a few tests. Contains the English names of all
+# integers between 1 and 300.
+#
+set numbers [normalize {
+  one two three four five six seven eight nine ten eleven twelve thirteen
+  fourteen fifteen sixteen seventeen eighteen nineteen twenty twentyone
+  twentytwo twentythree twentyfour twentyfive twentysix twentyseven
+  twentyeight twentynine thirty thirtyone thirtytwo thirtythree thirtyfour
+  thirtyfive thirtysix thirtyseven thirtyeight thirtynine forty fortyone
+  fortytwo fortythree fortyfour fortyfive fortysix fortyseven fortyeight
+  fortynine fifty fiftyone fiftytwo fiftythree fiftyfour fiftyfive fiftysix
+  fiftyseven fiftyeight fiftynine sixty sixtyone sixtytwo sixtythree sixtyfour
+  sixtyfive sixtysix sixtyseven sixtyeight sixtynine seventy seventyone
+  seventytwo seventythree seventyfour seventyfive seventysix seventyseven
+  seventyeight seventynine eighty eightyone eightytwo eightythree eightyfour
+  eightyfive eightysix eightyseven eightyeight eightynine ninety ninetyone
+  ninetytwo ninetythree ninetyfour ninetyfive ninetysix ninetyseven
+  ninetyeight ninetynine onehundred onehundredone onehundredtwo
+  onehundredthree onehundredfour onehundredfive onehundredsix onehundredseven
+  onehundredeight onehundrednine onehundredten onehundredeleven
+  onehundredtwelve onehundredthirteen onehundredfourteen onehundredfifteen
+  onehundredsixteen onehundredseventeen onehundredeighteen onehundrednineteen
+  onehundredtwenty onehundredtwentyone onehundredtwentytwo
+  onehundredtwentythree onehundredtwentyfour onehundredtwentyfive
+  onehundredtwentysix onehundredtwentyseven onehundredtwentyeight
+  onehundredtwentynine onehundredthirty onehundredthirtyone
+  onehundredthirtytwo onehundredthirtythree onehundredthirtyfour
+  onehundredthirtyfive onehundredthirtysix onehundredthirtyseven
+  onehundredthirtyeight onehundredthirtynine onehundredforty
+  onehundredfortyone onehundredfortytwo onehundredfortythree
+  onehundredfortyfour onehundredfortyfive onehundredfortysix
+  onehundredfortyseven onehundredfortyeight onehundredfortynine
+  onehundredfifty onehundredfiftyone onehundredfiftytwo onehundredfiftythree
+  onehundredfiftyfour onehundredfiftyfive onehundredfiftysix
+  onehundredfiftyseven onehundredfiftyeight onehundredfiftynine
+  onehundredsixty onehundredsixtyone onehundredsixtytwo onehundredsixtythree
+  onehundredsixtyfour onehundredsixtyfive onehundredsixtysix
+  onehundredsixtyseven onehundredsixtyeight onehundredsixtynine
+  onehundredseventy onehundredseventyone onehundredseventytwo
+  onehundredseventythree onehundredseventyfour onehundredseventyfive
+  onehundredseventysix onehundredseventyseven onehundredseventyeight
+  onehundredseventynine onehundredeighty onehundredeightyone
+  onehundredeightytwo onehundredeightythree onehundredeightyfour
+  onehundredeightyfive onehundredeightysix onehundredeightyseven
+  onehundredeightyeight onehundredeightynine onehundredninety
+  onehundredninetyone onehundredninetytwo onehundredninetythree
+  onehundredninetyfour onehundredninetyfive onehundredninetysix
+  onehundredninetyseven onehundredninetyeight onehundredninetynine twohundred
+  twohundredone twohundredtwo twohundredthree twohundredfour twohundredfive
+  twohundredsix twohundredseven twohundredeight twohundrednine twohundredten
+  twohundredeleven twohundredtwelve twohundredthirteen twohundredfourteen
+  twohundredfifteen twohundredsixteen twohundredseventeen twohundredeighteen
+  twohundrednineteen twohundredtwenty twohundredtwentyone twohundredtwentytwo
+  twohundredtwentythree twohundredtwentyfour twohundredtwentyfive
+  twohundredtwentysix twohundredtwentyseven twohundredtwentyeight
+  twohundredtwentynine twohundredthirty twohundredthirtyone
+  twohundredthirtytwo twohundredthirtythree twohundredthirtyfour
+  twohundredthirtyfive twohundredthirtysix twohundredthirtyseven
+  twohundredthirtyeight twohundredthirtynine twohundredforty
+  twohundredfortyone twohundredfortytwo twohundredfortythree
+  twohundredfortyfour twohundredfortyfive twohundredfortysix
+  twohundredfortyseven twohundredfortyeight twohundredfortynine
+  twohundredfifty twohundredfiftyone twohundredfiftytwo twohundredfiftythree
+  twohundredfiftyfour twohundredfiftyfive twohundredfiftysix
+  twohundredfiftyseven twohundredfiftyeight twohundredfiftynine
+  twohundredsixty twohundredsixtyone twohundredsixtytwo twohundredsixtythree
+  twohundredsixtyfour twohundredsixtyfive twohundredsixtysix
+  twohundredsixtyseven twohundredsixtyeight twohundredsixtynine
+  twohundredseventy twohundredseventyone twohundredseventytwo
+  twohundredseventythree twohundredseventyfour twohundredseventyfive
+  twohundredseventysix twohundredseventyseven twohundredseventyeight
+  twohundredseventynine twohundredeighty twohundredeightyone
+  twohundredeightytwo twohundredeightythree twohundredeightyfour
+  twohundredeightyfive twohundredeightysix twohundredeightyseven
+  twohundredeightyeight twohundredeightynine twohundredninety
+  twohundredninetyone twohundredninetytwo twohundredninetythree
+  twohundredninetyfour twohundredninetyfive twohundredninetysix
+  twohundredninetyseven twohundredninetyeight twohundredninetynine
+  threehundred
+}]
+
+foreach {DO_MALLOC_TEST enc} {
+  0 utf8
+  1 utf8
+  1 utf16
+} {
+
+  db close
+  forcedelete test.db
+  sqlite3 db test.db
+  sqlite3_db_config_lookaside db 0 0 0
+  db eval "PRAGMA encoding = \"$enc\""
+
+  # Set variable $T to the test name prefix for this iteration of the loop.
+  #
+  set T "fts3snippet-$enc"
+
+  ##########################################################################
+  # Test the offset function.
+  #
+  do_test $T.1.1 {
+    execsql {
+      CREATE VIRTUAL TABLE ft USING fts3;
+      INSERT INTO ft VALUES('xxx xxx xxx xxx');
+    }
+  } {}
+  do_offsets_test $T.1.2 {xxx} {0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3}
+  do_offsets_test $T.1.3 {"xxx xxx"} {
+      0 0  0 3     0 0  4 3     0 1  4 3     0 0  8 3 
+      0 1  8 3     0 1 12 3
+  }
+  do_offsets_test $T.1.4 {"xxx xxx" xxx} {
+      0 0  0 3     0 2  0 3     0 0  4 3     0 1  4 3 
+      0 2  4 3     0 0  8 3     0 1  8 3     0 2  8 3 
+      0 1 12 3     0 2 12 3
+  }
+  do_offsets_test $T.1.5 {xxx "xxx xxx"} {
+      0 0  0 3     0 1  0 3     0 0  4 3     0 1  4 3 
+      0 2  4 3     0 0  8 3     0 1  8 3     0 2  8 3 
+      0 0 12 3     0 2 12 3
+  }
+
+  do_test $T.2.1 {
+    set v1 [lrange $numbers 0 99]
+    execsql {
+      DROP TABLE IF EXISTS ft;
+      CREATE VIRTUAL TABLE ft USING fts3(a, b);
+      INSERT INTO ft VALUES($v1, $numbers);
+      INSERT INTO ft VALUES($v1, NULL);
+    }
+  } {}
+
+  set off [string first "twohundred " $numbers]
+  do_offsets_test $T.2.1 {twohundred} [list 1 0 $off 10]
+
+  set off [string first "onehundred " $numbers]
+  do_offsets_test $T.2.2 {onehundred} \
+    [list 0 0 $off 10 1 0 $off 10] [list 0 0 $off 10]
+
+  # Test a corruption case:
+  execsql { UPDATE ft_content SET c1b = 'hello world' WHERE c1b = $numbers }
+  do_error_test $T.2.3 {
+    SELECT offsets(ft) FROM ft WHERE ft MATCH 'onehundred'
+  } {database disk image is malformed}
+  
+  ##########################################################################
+  # Test the snippet function.
+  #
+  proc do_snippet_test {name expr iCol nTok args} {
+    set res [list]
+    foreach a $args { lappend res [string trim $a] }
+    do_select_test $name {
+      SELECT snippet(ft,'{','}','...',$iCol,$nTok) FROM ft WHERE ft MATCH $expr
+    } $res
+  }
+  do_test $T.3.1 {
+    execsql {
+      DROP TABLE IF EXISTS ft;
+      CREATE VIRTUAL TABLE ft USING fts3;
+      INSERT INTO ft VALUES('one two three four five six seven eight nine ten');
+    }
+  } {}
+  do_snippet_test $T.3.2  one    0 5 "{one} two three four five..."
+  do_snippet_test $T.3.3  two    0 5 "one {two} three four five..."
+  do_snippet_test $T.3.4  three  0 5 "one two {three} four five..."
+  do_snippet_test $T.3.5  four   0 5 "...two three {four} five six..."
+  do_snippet_test $T.3.6  five   0 5 "...three four {five} six seven..."
+  do_snippet_test $T.3.7  six    0 5 "...four five {six} seven eight..."
+  do_snippet_test $T.3.8  seven  0 5 "...five six {seven} eight nine..."
+  do_snippet_test $T.3.9  eight  0 5 "...six seven {eight} nine ten"
+  do_snippet_test $T.3.10 nine   0 5 "...six seven eight {nine} ten"
+  do_snippet_test $T.3.11 ten    0 5 "...six seven eight nine {ten}"
+  
+  do_test $T.4.1 {
+    execsql {
+      INSERT INTO ft VALUES(
+           'one two three four five '
+        || 'six seven eight nine ten '
+        || 'eleven twelve thirteen fourteen fifteen '
+        || 'sixteen seventeen eighteen nineteen twenty '
+        || 'one two three four five '
+        || 'six seven eight nine ten '
+        || 'eleven twelve thirteen fourteen fifteen '
+        || 'sixteen seventeen eighteen nineteen twenty'
+      );
+    }
+  } {}
+  
+  do_snippet_test $T.4.2 {one nine} 0 5 {
+     {one} two three...eight {nine} ten
+  } {
+     {one} two three...eight {nine} ten...
+  }
+  
+  do_snippet_test $T.4.3 {one nine} 0 -5 {
+     {one} two three four five...six seven eight {nine} ten
+  } {
+     {one} two three four five...seven eight {nine} ten eleven...
+  }
+  do_snippet_test $T.4.3 {one nineteen} 0 -5 {
+     ...eighteen {nineteen} twenty {one} two...
+  }
+  do_snippet_test $T.4.4 {two nineteen} 0 -5 {
+     ...eighteen {nineteen} twenty one {two}...
+  }
+  do_snippet_test $T.4.5 {three nineteen} 0 -5 {
+     ...{nineteen} twenty one two {three}...
+  }
+  
+  do_snippet_test $T.4.6 {four nineteen} 0 -5 {
+     ...two three {four} five six...seventeen eighteen {nineteen} twenty one...
+  }
+  do_snippet_test $T.4.7 {four NEAR nineteen} 0 -5 {
+     ...seventeen eighteen {nineteen} twenty one...two three {four} five six...
+  }
+  
+  do_snippet_test $T.4.8 {four nineteen} 0 5 {
+     ...three {four} five...eighteen {nineteen} twenty...
+  }
+  do_snippet_test $T.4.9 {four NEAR nineteen} 0 5 {
+     ...eighteen {nineteen} twenty...three {four} five...
+  }
+  do_snippet_test $T.4.10 {four NEAR nineteen} 0 -5 {
+     ...seventeen eighteen {nineteen} twenty one...two three {four} five six...
+  }
+  do_snippet_test $T.4.11 {four NOT (nineteen twentyone)} 0 5 {
+     ...two three {four} five six...
+  } {
+     ...two three {four} five six...
+  }
+  do_snippet_test $T.4.12 {four OR nineteen NEAR twentyone} 0 5 {
+     ...two three {four} five six...
+  } {
+     ...two three {four} five six...
+  }
+  
+  do_test $T.5.1 {
+    execsql {
+      DROP TABLE IF EXISTS ft;
+      CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
+      INSERT INTO ft VALUES(
+        'one two three four five', 
+        'four five six seven eight', 
+        'seven eight nine ten eleven'
+      );
+    }
+  } {}
+  
+  do_snippet_test $T.5.2 {five} -1 3 {...three four {five}}
+  do_snippet_test $T.5.3 {five}  0 3 {...three four {five}}
+  do_snippet_test $T.5.4 {five}  1 3 {four {five} six...}
+  do_snippet_test $T.5.5 {five}  2 3 {seven eight nine...}
+  
+  do_test $T.5.6 {
+    execsql { UPDATE ft SET b = NULL }
+  } {}
+  
+  do_snippet_test $T.5.7  {five} -1 3 {...three four {five}}
+  do_snippet_test $T.5.8  {five}  0 3 {...three four {five}}
+  do_snippet_test $T.5.9  {five}  1 3 {}
+  do_snippet_test $T.5.10 {five}  2 3 {seven eight nine...}
+  
+  do_snippet_test $T.5.11 {one "seven eight nine"} -1 -3 {
+    {one} two three...{seven} {eight} {nine}...
+  }
+
+  do_test $T.6.1 {
+    execsql {
+      DROP TABLE IF EXISTS ft;
+      CREATE VIRTUAL TABLE ft USING fts3(x);
+      INSERT INTO ft VALUES($numbers);
+    }
+  } {}
+  do_snippet_test $T.6.2 {
+    one fifty onehundred onehundredfifty twohundredfifty threehundred
+  } -1 4 {
+    {one}...{fifty}...{onehundred}...{onehundredfifty}...
+  }
+  do_snippet_test $T.6.3 {
+    one fifty onehundred onehundredfifty twohundredfifty threehundred
+  } -1 -4 {
+    {one} two three four...fortyeight fortynine {fifty} fiftyone...ninetyeight ninetynine {onehundred} onehundredone...onehundredfortyeight onehundredfortynine {onehundredfifty} onehundredfiftyone...
+  }
+
+  do_test $T.7.1 {
+    execsql {
+      BEGIN;
+        DROP TABLE IF EXISTS ft;
+        CREATE VIRTUAL TABLE ft USING fts3(x);
+    }
+    set testresults [list]
+    for {set i 1} {$i < 150} {incr i} {
+      set commas [string repeat , $i]
+      execsql {INSERT INTO ft VALUES('one' || $commas || 'two')}
+      lappend testresults "{one}$commas{two}"
+    }
+    execsql COMMIT
+  } {}
+  eval [list do_snippet_test $T.7.2 {one two} -1 3] $testresults
+  
+  ##########################################################################
+  # Test the matchinfo function.
+  #
+  proc mit {blob} {
+    set scan(littleEndian) i*
+    set scan(bigEndian) I*
+    binary scan $blob $scan($::tcl_platform(byteOrder)) r
+    return $r
+  }
+  db func mit mit
+  proc do_matchinfo_test {name expr args} {
+    set res [list]
+    foreach a $args { lappend res [normalize $a] }
+    do_select_test $name {
+      SELECT mit(matchinfo(ft)) FROM ft WHERE ft MATCH $expr
+    } $res
+  }
+  do_test $T.8.1 {
+    set ten {one two three four five six seven eight nine ten}
+    execsql {
+      DROP TABLE IF EXISTS ft;
+      CREATE VIRTUAL TABLE ft USING fts3;
+      INSERT INTO ft VALUES($ten);
+      INSERT INTO ft VALUES($ten || ' ' || $ten);
+    }
+  } {}
+  
+  do_matchinfo_test $T.8.2 "one" {1 1  1 3 2} {1 1  2 3 2}
+  do_matchinfo_test $T.8.3 "one NEAR/3 ten" {2 1  1 1 1 1 1 1}
+  do_matchinfo_test $T.8.4 "five NEAR/4 ten" \
+    {2 1  1 3 2  1 3 2} {2 1  2 3 2  2 3 2}
+  do_matchinfo_test $T.8.5 "six NEAR/3 ten NEAR/3 two" \
+    {3 1  1 1 1  1 1 1  1 1 1}
+  do_matchinfo_test $T.8.6 "five NEAR/4 ten NEAR/3 two" \
+    {3 1  2 2 1  1 1 1  1 1 1}
+
+  do_test $T.9.1 {
+    execsql {
+      DROP TABLE IF EXISTS ft;
+      CREATE VIRTUAL TABLE ft USING fts3(x, y);
+    }
+    foreach n {1 2 3} {
+      set v1 [lrange $numbers 0 [expr $n*100]]
+      set v2 [string trim [string repeat "$numbers " $n]]
+      set docid [expr $n * 1000000]
+      execsql { INSERT INTO ft(docid, x, y) VALUES($docid, $v1, $v2) }
+    }
+  } {}
+  do_matchinfo_test $T.9.2 {two*}     \
+    { 1 2    1   105 3   101 606 3}   \
+    { 1 2    3   105 3   202 606 3}   \
+    { 1 2    101 105 3   303 606 3}
+
+  do_matchinfo_test $T.9.4 {"one* two*"}  \
+    { 1 2    1 5 3   2 12 3}              \
+    { 1 2    2 5 3   4 12 3}              \
+    { 1 2    2 5 3   6 12 3}
+
+  do_matchinfo_test $T.9.5 {twohundredfifty}  \
+    { 1 2    0 1 1   1 6 3}                   \
+    { 1 2    0 1 1   2 6 3}                   \
+    { 1 2    1 1 1   3 6 3}
+
+  do_matchinfo_test $T.9.6 {"threehundred one"} \
+    { 1 2    0 0 0   1 3 2}                     \
+    { 1 2    0 0 0   2 3 2}
+
+  do_matchinfo_test $T.9.7 {one OR fivehundred} \
+    { 2 2    1 3 3   1 6 3   0 0 0   0 0 0 }    \
+    { 2 2    1 3 3   2 6 3   0 0 0   0 0 0 }    \
+    { 2 2    1 3 3   3 6 3   0 0 0   0 0 0 }
+
+  do_matchinfo_test $T.9.8 {two OR "threehundred one"} \
+    { 2 2    1 3 3   1 6 3   0 0 0   0 3 2 }           \
+    { 2 2    1 3 3   2 6 3   0 0 0   1 3 2 }           \
+    { 2 2    1 3 3   3 6 3   0 0 0   2 3 2 }
+
+  do_select_test $T.9.9 {
+    SELECT mit(matchinfo(ft)), mit(matchinfo(ft))
+    FROM ft WHERE ft MATCH 'two OR "threehundred one"' 
+  } [normalize {
+    {2 2 1 3 3 1 6 3 0 0 0 0 3 2}
+    {2 2 1 3 3 1 6 3 0 0 0 0 3 2}
+    {2 2 1 3 3 2 6 3 0 0 0 1 3 2}
+    {2 2 1 3 3 2 6 3 0 0 0 1 3 2}
+    {2 2 1 3 3 3 6 3 0 0 0 2 3 2}          
+    {2 2 1 3 3 3 6 3 0 0 0 2 3 2}
+  }]
+
+  # EVIDENCE-OF: R-40630-02268 If used within a SELECT that uses the

+  # "query by rowid" or "linear scan" strategies, then the snippet and

+  # offsets both return an empty string, and the matchinfo function

+  # returns a blob value zero bytes in size.

+  #
+  set r 1000000                   ;# A rowid that exists in table ft
+  do_select_test $T.10.0 { SELECT rowid FROM ft WHERE rowid = $r } $r
+  do_select_test $T.10.1 {
+    SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft;
+  } {0 text 0 text 0 text}
+  do_select_test $T.10.2 {
+    SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft WHERE rowid = $r
+  } {0 text}
+  do_select_test $T.10.3 {
+    SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft;
+  } {0 text 0 text 0 text}
+  do_select_test $T.10.4 {
+    SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft WHERE rowid = $r;
+  } {0 text}
+  do_select_test $T.10.5 {
+    SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft;
+  } {0 blob 0 blob 0 blob}
+  do_select_test $T.10.6 {
+    SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft WHERE rowid = $r
+  } {0 blob}
+}
+
+set sqlite_fts3_enable_parentheses 0
+finish_test