Merge remote-tracking branch 'kali/bug/imap-service-use-defer-to-thread' into develop

6 files changed, 256 insertions, 149 deletions

diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py
index 60ae387..df5d046 100644
--- a/mail/src/leap/mail/imap/fetch.py
+++ b/mail/src/leap/mail/imap/fetch.py
@@ -1,17 +1,31 @@
+import logging
 import json
 
 from twisted.python import log
+from twisted.internet import defer
+from twisted.internet.threads import deferToThread
 
 from leap.common.check import leap_assert, leap_assert_type
 from leap.soledad import Soledad
 
 from leap.common.keymanager import openpgp
 
+logger = logging.getLogger(__name__)
+
 
 class LeapIncomingMail(object):
     """
     Fetches mail from the incoming queue.
     """
+
+    ENC_SCHEME_KEY = "_enc_scheme"
+    ENC_JSON_KEY = "_enc_json"
+
+    RECENT_FLAG = "\\Recent"
+
+    INCOMING_KEY = "incoming"
+    CONTENT_KEY = "content"
+
     def __init__(self, keymanager, soledad, imap_account):
 
         """
@@ -33,57 +47,89 @@ class LeapIncomingMail(object):
         self._keymanager = keymanager
         self._soledad = soledad
         self.imapAccount = imap_account
+        self._inbox = self.imapAccount.getMailbox('inbox')
 
         self._pkey = self._keymanager.get_all_keys_in_local_db(
             private=True).pop()
 
     def fetch(self):
         """
-        Get new mail by syncing database, store it in the INBOX for the
-        user account, and remove from the incoming db.
+        Fetch incoming mail, to be called periodically.
+
+        Calls a deferred that will execute the fetch callback
+        in a separate thread
         """
+        logger.debug('fetching mail...')
+        d = deferToThread(self._sync_soledad)
+        d.addCallbacks(self._process_doclist, self._sync_soledad_err)
+        return d
+
+    def _sync_soledad(self):
+        log.msg('syncing soledad...')
+        logger.debug('in soledad sync')
+        #import ipdb; ipdb.set_trace()
+
         self._soledad.sync()
         gen, doclist = self._soledad.get_all_docs()
-        #log.msg("there are %s docs" % (len(doclist),))
+        #logger.debug("there are %s docs" % (len(doclist),))
+        log.msg("there are %s docs" % (len(doclist),))
+        return doclist
 
-        if doclist:
-            inbox = self.imapAccount.getMailbox('inbox')
+    def _sync_soledad_err(self, f):
+        log.err("error syncing soledad: %s" % (f.value,))
+        return f
 
-        key = self._pkey
+    def _process_doclist(self, doclist):
+        log.msg('processing doclist')
         for doc in doclist:
             keys = doc.content.keys()
-            if '_enc_scheme' in keys and '_enc_json' in keys:
+            if self.ENC_SCHEME_KEY in keys and self.ENC_JSON_KEY in keys:
 
                 # XXX should check for _enc_scheme == "pubkey" || "none"
                 # that is what incoming mail uses.
+                encdata = doc.content[self.ENC_JSON_KEY]
+                d = defer.Deferred(self._decrypt_msg, doc, encdata)
+                d.addCallback(self._process_decrypted)
 
-                encdata = doc.content['_enc_json']
-                decrdata = openpgp.decrypt_asym(
-                    encdata, key,
-                    # XXX get from public method instead
-                    passphrase=self._soledad._passphrase)
-                if decrdata:
-                    self.process_decrypted(doc, decrdata, inbox)
-        # XXX launch sync callback / defer
+    def _decrypt_msg(self, doc, encdata):
+        log.msg('decrypting msg')
+        key = self._pkey
+        decrdata = (openpgp.decrypt_asym(
+            encdata, key,
+            # XXX get from public method instead
+            passphrase=self._soledad._passphrase))
+        return doc, decrdata
 
-    def process_decrypted(self, doc, data, inbox):
+    def _process_decrypted(self, doc, data):
         """
-        Process a successfully decrypted message
+        Process a successfully decrypted message.
+
+        :param doc: a LeapDocument instance containing the incoming message
+        :type doc: LeapDocument
+
+        :param data: the json-encoded, decrypted content of the incoming
+                     message
+        :type data: str
+
+        :param inbox: a open SoledadMailbox instance where this message is
+                      to be saved
+        :type inbox: SoledadMailbox
         """
         log.msg("processing incoming message!")
         msg = json.loads(data)
         if not isinstance(msg, dict):
             return False
-        if not msg.get('incoming', False):
+        if not msg.get(self.INCOMING_KEY, False):
             return False
         # ok, this is an incoming message
-        rawmsg = msg.get('content', None)
+        rawmsg = msg.get(self.CONTENT_KEY, None)
         if not rawmsg:
             return False
+        logger.debug('got incoming message: %s' % (rawmsg,))
         #log.msg("we got raw message")
 
         # add to inbox and delete from soledad
-        inbox.addMessage(rawmsg, ("\\Recent",))
+        self.inbox.addMessage(rawmsg, (self.RECENT_FLAG,))
         doc_id = doc.doc_id
         self._soledad.delete_doc(doc)
         log.msg("deleted doc %s from incoming" % doc_id)
diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py
index 30938db..45c43b7 100644
--- a/mail/src/leap/mail/imap/server.py
+++ b/mail/src/leap/mail/imap/server.py
@@ -814,8 +814,11 @@ class MessageCollection(WithMsgFields):
         leap_assert(isinstance(mbox, (str, unicode)),
                     "mbox needs to be a string")
         leap_assert(soledad, "Need a soledad instance to initialize")
-        leap_assert(isinstance(soledad._db, SQLCipherDatabase),
-                    "soledad._db must be an instance of SQLCipherDatabase")
+
+        # This is a wrapper now!...
+        # should move assertion there...
+        #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
+                    #"soledad._db must be an instance of SQLCipherDatabase")
 
         # okay, all in order, keep going...
 
@@ -1080,8 +1083,10 @@ class SoledadMailbox(WithMsgFields):
         """
         leap_assert(mbox, "Need a mailbox name to initialize")
         leap_assert(soledad, "Need a soledad instance to initialize")
-        leap_assert(isinstance(soledad._db, SQLCipherDatabase),
-                    "soledad._db must be an instance of SQLCipherDatabase")
+
+        # XXX should move to wrapper
+        #leap_assert(isinstance(soledad._db, SQLCipherDatabase),
+                    #"soledad._db must be an instance of SQLCipherDatabase")
 
         self.mbox = mbox
         self.rw = rw
diff --git a/mail/src/leap/mail/imap/service/__init__.py b/mail/src/leap/mail/imap/service/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/mail/src/leap/mail/imap/service/__init__.py
diff --git a/mail/src/leap/mail/imap/service/imap-server.tac b/mail/src/leap/mail/imap/service/imap-server.tac
index 1a4661b..16d04bb 100644
--- a/mail/src/leap/mail/imap/service/imap-server.tac
+++ b/mail/src/leap/mail/imap/service/imap-server.tac
@@ -3,99 +3,21 @@ import os
 
 from xdg import BaseDirectory
 
-from twisted.application import internet, service
-from twisted.internet.protocol import ServerFactory
-from twisted.mail import imap4
-from twisted.python import log
-
-from leap.common.check import leap_assert, leap_assert_type
-from leap.mail.imap.server import SoledadBackedAccount
-from leap.mail.imap.fetch import LeapIncomingMail
 from leap.soledad import Soledad
+from leap.mail.imap.service import imap
 
-# Some constants
-# XXX Should be passed to initializer too.
 
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-IMAP_PORT = 9930
-# The port in which imap service will run
-
-INCOMING_CHECK_PERIOD = 10
-# The period between succesive checks of the incoming mail
-# queue (in seconds)
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-
-class LeapIMAPServer(imap4.IMAP4Server):
-    """
-    An IMAP4 Server with mailboxes backed by soledad
-    """
-    def __init__(self, *args, **kwargs):
-        # pop extraneous arguments
-        soledad = kwargs.pop('soledad', None)
-        user = kwargs.pop('user', None)
-        leap_assert(soledad, "need a soledad instance")
-        leap_assert_type(soledad, Soledad)
-        leap_assert(user, "need a user in the initialization")
-
-        # initialize imap server!
-        imap4.IMAP4Server.__init__(self, *args, **kwargs)
-
-        # we should initialize the account here,
-        # but we move it to the factory so we can
-        # populate the test account properly (and only once
-        # per session)
-
-        # theAccount = SoledadBackedAccount(
-        #     user, soledad=soledad)
-
-        # ---------------------------------
-        # XXX pre-populate acct for tests!!
-        # populate_test_account(theAccount)
-        # ---------------------------------
-        #self.theAccount = theAccount
-
-    def lineReceived(self, line):
-        log.msg('rcv: %s' % line)
-        imap4.IMAP4Server.lineReceived(self, line)
-
-    def authenticateLogin(self, username, password):
-        # all is allowed so far. use realm instead
-        return imap4.IAccount, self.theAccount, lambda: None
-
-
-class IMAPAuthRealm(object):
-    """
-    Dummy authentication realm. Do not use in production!
-    """
-    theAccount = None
-
-    def requestAvatar(self, avatarId, mind, *interfaces):
-        return imap4.IAccount, self.theAccount, lambda: None
-
-
-class LeapIMAPFactory(ServerFactory):
-    """
-    Factory for a IMAP4 server with soledad remote sync and gpg-decryption
-    capabilities.
-    """
+config = ConfigParser.ConfigParser()
+config.read([os.path.expanduser('~/.config/leap/mail/mail.conf')])
 
-    def __init__(self, user, soledad):
-        self._user = user
-        self._soledad = soledad
+userID = config.get('mail', 'address')
+privkey = open(os.path.expanduser('~/.config/leap/mail/privkey')).read()
+nickserver_url = ""
 
-        theAccount = SoledadBackedAccount(
-            user, soledad=soledad)
-        self.theAccount = theAccount
+d = {}
 
-    def buildProtocol(self, addr):
-        "Return a protocol suitable for the job."
-        imapProtocol = LeapIMAPServer(
-            user=self._user,
-            soledad=self._soledad)
-        imapProtocol.theAccount = self.theAccount
-        imapProtocol.factory = self
-        return imapProtocol
+for key in ('uid', 'passphrase', 'server', 'pemfile', 'token'):
+    d[key] = config.get('mail', key)
 
 
 def initialize_soledad_mailbox(user_uuid, soledad_pass, server_url,
@@ -113,6 +35,7 @@ def initialize_soledad_mailbox(user_uuid, soledad_pass, server_url,
     """
 
     base_config = BaseDirectory.xdg_config_home
+
     secret_path = os.path.join(
         base_config, "leap", "soledad", "%s.secret" % user_uuid)
     soledad_path = os.path.join(
@@ -129,22 +52,6 @@ def initialize_soledad_mailbox(user_uuid, soledad_pass, server_url,
 
     return _soledad
 
-
-#######################################################################
-# XXX STUBBED! We need to get this in the instantiation from the client
-
-config = ConfigParser.ConfigParser()
-config.read([os.path.expanduser('~/.config/leap/mail/mail.conf')])
-
-userID = config.get('mail', 'address')
-privkey = open(os.path.expanduser('~/.config/leap/mail/privkey')).read()
-nickserver_url = ""
-
-d = {}
-
-for key in ('uid', 'passphrase', 'server', 'pemfile', 'token'):
-    d[key] = config.get('mail', key)
-
 soledad = initialize_soledad_mailbox(
     d['uid'],
     d['passphrase'],
@@ -158,21 +65,6 @@ opgp = OpenPGPScheme(soledad)
 opgp.put_ascii_key(privkey)
 
 from leap.common.keymanager import KeyManager
-keym = KeyManager(userID, nickserver_url, soledad, d['token'])
-
-
-factory = LeapIMAPFactory(userID, soledad)
-
-application = service.Application("LEAP IMAP4 Local Service")
-imapService = internet.TCPServer(IMAP_PORT, factory)
-imapService.setServiceParent(application)
-
-fetcher = LeapIncomingMail(
-    keym,
-    soledad,
-    factory.theAccount)
-
+keymanager = KeyManager(userID, nickserver_url, soledad, d['token'])
 
-internet.TimerService(
-    INCOMING_CHECK_PERIOD,
-    fetcher.fetch).setServiceParent(application)
+imap.run_service(soledad, keymanager)
diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py
new file mode 100644
index 0000000..49d54e3
--- /dev/null
+++ b/mail/src/leap/mail/imap/service/imap.py
@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+# imap.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Imap service initialization
+"""
+import logging
+logger = logging.getLogger(__name__)
+
+#from twisted.application import internet, service
+from twisted.internet.protocol import ServerFactory
+from twisted.internet.task import LoopingCall
+
+from twisted.mail import imap4
+from twisted.python import log
+
+from leap.common.check import leap_assert, leap_assert_type
+from leap.common.keymanager import KeyManager
+from leap.mail.imap.server import SoledadBackedAccount
+from leap.mail.imap.fetch import LeapIncomingMail
+from leap.soledad import Soledad
+
+IMAP_PORT = 9930
+# The default port in which imap service will run
+
+#INCOMING_CHECK_PERIOD = 10
+INCOMING_CHECK_PERIOD = 5
+# The period between succesive checks of the incoming mail
+# queue (in seconds)
+
+
+class LeapIMAPServer(imap4.IMAP4Server):
+    """
+    An IMAP4 Server with mailboxes backed by soledad
+    """
+    def __init__(self, *args, **kwargs):
+        # pop extraneous arguments
+        soledad = kwargs.pop('soledad', None)
+        user = kwargs.pop('user', None)
+        leap_assert(soledad, "need a soledad instance")
+        leap_assert_type(soledad, Soledad)
+        leap_assert(user, "need a user in the initialization")
+
+        # initialize imap server!
+        imap4.IMAP4Server.__init__(self, *args, **kwargs)
+
+        # we should initialize the account here,
+        # but we move it to the factory so we can
+        # populate the test account properly (and only once
+        # per session)
+
+        # theAccount = SoledadBackedAccount(
+        #     user, soledad=soledad)
+
+        # ---------------------------------
+        # XXX pre-populate acct for tests!!
+        # populate_test_account(theAccount)
+        # ---------------------------------
+        #self.theAccount = theAccount
+
+    def lineReceived(self, line):
+        log.msg('rcv: %s' % line)
+        imap4.IMAP4Server.lineReceived(self, line)
+
+    def authenticateLogin(self, username, password):
+        # all is allowed so far. use realm instead
+        return imap4.IAccount, self.theAccount, lambda: None
+
+
+class IMAPAuthRealm(object):
+    """
+    Dummy authentication realm. Do not use in production!
+    """
+    theAccount = None
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        return imap4.IAccount, self.theAccount, lambda: None
+
+
+class LeapIMAPFactory(ServerFactory):
+    """
+    Factory for a IMAP4 server with soledad remote sync and gpg-decryption
+    capabilities.
+    """
+
+    def __init__(self, user, soledad):
+        """
+        Initializes the server factory.
+
+        :param user: user ID. **right now it's uuid**
+                     this might change!
+        :type user: str
+
+        :param soledad: soledad instance
+        :type soledad: Soledad
+        """
+        self._user = user
+        self._soledad = soledad
+
+        theAccount = SoledadBackedAccount(
+            user, soledad=soledad)
+        self.theAccount = theAccount
+
+    def buildProtocol(self, addr):
+        "Return a protocol suitable for the job."
+        imapProtocol = LeapIMAPServer(
+            user=self._user,
+            soledad=self._soledad)
+        imapProtocol.theAccount = self.theAccount
+        imapProtocol.factory = self
+        return imapProtocol
+
+
+def run_service(*args, **kwargs):
+    """
+    Main entry point to run the service from the client.
+    """
+    leap_assert(len(args) == 2)
+    soledad, keymanager = args
+    leap_assert_type(soledad, Soledad)
+    leap_assert_type(keymanager, KeyManager)
+
+    port = kwargs.get('port', IMAP_PORT)
+    check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD)
+
+    uuid = soledad._get_uuid()
+    factory = LeapIMAPFactory(uuid, soledad)
+
+    # ---- for application framework
+    #application = service.Application("LEAP IMAP4 Local Service")
+    #imapService = internet.TCPServer(port, factory)
+    #imapService.setServiceParent(application)
+
+    from twisted.internet import reactor
+    reactor.listenTCP(port, factory)
+
+    fetcher = LeapIncomingMail(
+        keymanager,
+        soledad,
+        factory.theAccount)
+
+    lc = LoopingCall(fetcher.fetch)
+    lc.start(check_period)
+
+    # ---- for application framework
+    #internet.TimerService(
+        #check_period,
+        #fetcher.fetch).setServiceParent(application)
+
+    logger.debug('----------------------------------------')
+    logger.debug("IMAP4 Server is RUNNING in port  %s" % (port,))
+
+    #log.msg("IMAP4 Server is RUNNING in port  %s" % (port,))
+    #return application
diff --git a/mail/src/leap/mail/smtp/__init__.py b/mail/src/leap/mail/smtp/__init__.py
index ace79b5..daa7ccf 100644
--- a/mail/src/leap/mail/smtp/__init__.py
+++ b/mail/src/leap/mail/smtp/__init__.py
@@ -15,18 +15,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
-
 """
 SMTP relay helper function.
 """
 
-
-from twisted.application import internet, service
 from twisted.internet import reactor
 
 
-from leap import soledad
-from leap.common.keymanager import KeyManager
+#from leap import soledad
+#from leap.common.keymanager import KeyManager
 from leap.mail.smtp.smtprelay import SMTPFactory