[feat] async index writer to avoid locking #404

This commit changes index writer to use AsyncWriter.
It should avoid locking on our code and speed up things for now.

2 files changed, 7 insertions, 36 deletions

diff --git a/service/pixelated/adapter/search/__init__.py b/service/pixelated/adapter/search/__init__.py
index 91eff4c3..da8845cc 100644
--- a/service/pixelated/adapter/search/__init__.py
+++ b/service/pixelated/adapter/search/__init__.py
@@ -24,10 +24,10 @@ from whoosh.index import FileIndex
 from whoosh.fields import Schema, ID, KEYWORD, TEXT, NUMERIC
 from whoosh.qparser import QueryParser
 from whoosh.qparser import MultifieldParser
+from whoosh.writing import AsyncWriter
 from whoosh import sorting
 from pixelated.support.functional import unique
 from pixelated.support.date import milliseconds
-from threading import Lock
 import traceback
 
 
@@ -41,7 +41,6 @@ class SearchEngine(object):
         if not os.path.exists(self.index_folder):
             os.makedirs(self.index_folder)
         self._index = self._create_index()
-        self._write_lock = Lock()
 
     def _add_to_tags(self, tags, group, skip_default_tags, count_type, query=None):
         query_matcher = re.compile(".*%s.*" % query.lower()) if query else re.compile(".*")
@@ -117,8 +116,7 @@ class SearchEngine(object):
         return FileIndex.create(storage, self._mail_schema(), indexname='mails')
 
     def index_mail(self, mail):
-        with self._write_lock:
-            with self._index.writer() as writer:
+        with AsyncWriter(self._index) as writer:
                 self._index_mail(writer, mail)
 
     def _index_mail(self, writer, mail):
@@ -153,10 +151,9 @@ class SearchEngine(object):
 
     def index_mails(self, mails, callback=None):
         try:
-            with self._write_lock:
-                with self._index.writer() as writer:
-                    for mail in mails:
-                        self._index_mail(writer, mail)
+            with AsyncWriter(self._index) as writer:
+                for mail in mails:
+                    self._index_mail(writer, mail)
             if callback:
                 callback()
         except Exception, e:
@@ -198,12 +195,8 @@ class SearchEngine(object):
         return MultifieldParser(['raw', 'body'], self._index.schema).parse(query)
 
     def remove_from_index(self, mail_id):
-        with self._write_lock:
-            writer = self._index.writer()
-            try:
-                writer.delete_by_term('ident', mail_id)
-            finally:
-                writer.commit()
+        with AsyncWriter(self._index) as writer:
+            writer.delete_by_term('ident', mail_id)
 
     def contacts(self, query):
         with self._index.searcher() as searcher:
diff --git a/service/test/unit/adapter/search/test_search.py b/service/test/unit/adapter/search/test_search.py
index d57b8227..491cb2fc 100644
--- a/service/test/unit/adapter/search/test_search.py
+++ b/service/test/unit/adapter/search/test_search.py
@@ -45,28 +45,6 @@ class SearchEngineTest(unittest.TestCase):
     def tearDown(self):
         self.tempdir.dissolve()
 
-    def test_index_mail_secured_by_lock(self):
-        # given
-        soledad_querier = mock()
-        lock_stub = LockStub()
-        when(soledad_querier).get_index_masterkey().thenReturn(INDEX_KEY)
-
-        self.assertEqual(INDEX_KEY, soledad_querier.get_index_masterkey())
-        se = SearchEngine(soledad_querier, self.agent_home)
-        se._write_lock = lock_stub
-
-        headers = {
-            'From': 'from@bar.tld',
-            'To': 'to@bar.tld',
-            'Subject': 'Some test mail',
-        }
-
-        # when
-        se.index_mail(test_helper.pixelated_mail(extra_headers=headers))
-
-        # then
-        self.assertTrue(lock_stub.called)
-
     def test_encoding(self):
         # given
         soledad_querier = mock()