summaryrefslogtreecommitdiff
path: root/service/pixelated/adapter/search
diff options
context:
space:
mode:
authorVictor Shyba <victor.shyba@gmail.com>2015-06-04 21:27:01 -0300
committerVictor Shyba <victor.shyba@gmail.com>2015-06-04 21:27:01 -0300
commit7fe97788b27704015b13f097026fb7e1e44f5d90 (patch)
tree013936834668bd211f46a522a7c8c1fc575e0a9d /service/pixelated/adapter/search
parent9bcc2532f6974af2b6daad1096a0258e3a65e357 (diff)
[feat] async index writer to avoid locking #404
This commit changes index writer to use AsyncWriter. It should avoid locking on our code and speed up things for now.
Diffstat (limited to 'service/pixelated/adapter/search')
-rw-r--r--service/pixelated/adapter/search/__init__.py21
1 files changed, 7 insertions, 14 deletions
diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py
index 91eff4c3..da8845cc 100644
--- a/service/pixelated/adapter/search/__init__.py
+++ b/service/pixelated/adapter/search/__init__.py
@@ -24,10 +24,10 @@ from whoosh.index import FileIndex
from whoosh.fields import Schema, ID, KEYWORD, TEXT, NUMERIC
from whoosh.qparser import QueryParser
from whoosh.qparser import MultifieldParser
+from whoosh.writing import AsyncWriter
from whoosh import sorting
from pixelated.support.functional import unique
from pixelated.support.date import milliseconds
-from threading import Lock
import traceback
@@ -41,7 +41,6 @@ class SearchEngine(object):
if not os.path.exists(self.index_folder):
os.makedirs(self.index_folder)
self._index = self._create_index()
- self._write_lock = Lock()
def _add_to_tags(self, tags, group, skip_default_tags, count_type, query=None):
query_matcher = re.compile(".*%s.*" % query.lower()) if query else re.compile(".*")
@@ -117,8 +116,7 @@ class SearchEngine(object):
return FileIndex.create(storage, self._mail_schema(), indexname='mails')
def index_mail(self, mail):
- with self._write_lock:
- with self._index.writer() as writer:
+ with AsyncWriter(self._index) as writer:
self._index_mail(writer, mail)
def _index_mail(self, writer, mail):
@@ -153,10 +151,9 @@ class SearchEngine(object):
def index_mails(self, mails, callback=None):
try:
- with self._write_lock:
- with self._index.writer() as writer:
- for mail in mails:
- self._index_mail(writer, mail)
+ with AsyncWriter(self._index) as writer:
+ for mail in mails:
+ self._index_mail(writer, mail)
if callback:
callback()
except Exception, e:
@@ -198,12 +195,8 @@ class SearchEngine(object):
return MultifieldParser(['raw', 'body'], self._index.schema).parse(query)
def remove_from_index(self, mail_id):
- with self._write_lock:
- writer = self._index.writer()
- try:
- writer.delete_by_term('ident', mail_id)
- finally:
- writer.commit()
+ with AsyncWriter(self._index) as writer:
+ writer.delete_by_term('ident', mail_id)
def contacts(self, query):
with self._index.searcher() as searcher: