From 7fe97788b27704015b13f097026fb7e1e44f5d90 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 4 Jun 2015 21:27:01 -0300 Subject: [feat] async index writer to avoid locking #404 This commit changes index writer to use AsyncWriter. It should avoid locking on our code and speed up things for now. --- service/pixelated/adapter/search/__init__.py | 21 +++++++-------------- service/test/unit/adapter/search/test_search.py | 22 ---------------------- 2 files changed, 7 insertions(+), 36 deletions(-) (limited to 'service') diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py index 91eff4c3..da8845cc 100644 --- a/service/pixelated/adapter/search/__init__.py +++ b/service/pixelated/adapter/search/__init__.py @@ -24,10 +24,10 @@ from whoosh.index import FileIndex from whoosh.fields import Schema, ID, KEYWORD, TEXT, NUMERIC from whoosh.qparser import QueryParser from whoosh.qparser import MultifieldParser +from whoosh.writing import AsyncWriter from whoosh import sorting from pixelated.support.functional import unique from pixelated.support.date import milliseconds -from threading import Lock import traceback @@ -41,7 +41,6 @@ class SearchEngine(object): if not os.path.exists(self.index_folder): os.makedirs(self.index_folder) self._index = self._create_index() - self._write_lock = Lock() def _add_to_tags(self, tags, group, skip_default_tags, count_type, query=None): query_matcher = re.compile(".*%s.*" % query.lower()) if query else re.compile(".*") @@ -117,8 +116,7 @@ class SearchEngine(object): return FileIndex.create(storage, self._mail_schema(), indexname='mails') def index_mail(self, mail): - with self._write_lock: - with self._index.writer() as writer: + with AsyncWriter(self._index) as writer: self._index_mail(writer, mail) def _index_mail(self, writer, mail): @@ -153,10 +151,9 @@ class SearchEngine(object): def index_mails(self, mails, callback=None): try: - with self._write_lock: - with self._index.writer() as writer: - for mail in mails: - self._index_mail(writer, mail) + with AsyncWriter(self._index) as writer: + for mail in mails: + self._index_mail(writer, mail) if callback: callback() except Exception, e: @@ -198,12 +195,8 @@ class SearchEngine(object): return MultifieldParser(['raw', 'body'], self._index.schema).parse(query) def remove_from_index(self, mail_id): - with self._write_lock: - writer = self._index.writer() - try: - writer.delete_by_term('ident', mail_id) - finally: - writer.commit() + with AsyncWriter(self._index) as writer: + writer.delete_by_term('ident', mail_id) def contacts(self, query): with self._index.searcher() as searcher: diff --git a/service/test/unit/adapter/search/test_search.py b/service/test/unit/adapter/search/test_search.py index d57b8227..491cb2fc 100644 --- a/service/test/unit/adapter/search/test_search.py +++ b/service/test/unit/adapter/search/test_search.py @@ -45,28 +45,6 @@ class SearchEngineTest(unittest.TestCase): def tearDown(self): self.tempdir.dissolve() - def test_index_mail_secured_by_lock(self): - # given - soledad_querier = mock() - lock_stub = LockStub() - when(soledad_querier).get_index_masterkey().thenReturn(INDEX_KEY) - - self.assertEqual(INDEX_KEY, soledad_querier.get_index_masterkey()) - se = SearchEngine(soledad_querier, self.agent_home) - se._write_lock = lock_stub - - headers = { - 'From': 'from@bar.tld', - 'To': 'to@bar.tld', - 'Subject': 'Some test mail', - } - - # when - se.index_mail(test_helper.pixelated_mail(extra_headers=headers)) - - # then - self.assertTrue(lock_stub.called) - def test_encoding(self): # given soledad_querier = mock() -- cgit v1.2.3