Merge branch 'feature/async-api' into develop

8 files changed, 191 insertions, 159 deletions

diff --git a/src/leap/bitmask/backend/components.py b/src/leap/bitmask/backend/components.py
index 4b63af84..2ae796f6 100644
--- a/src/leap/bitmask/backend/components.py
+++ b/src/leap/bitmask/backend/components.py
@@ -17,13 +17,16 @@
 """
 Backend components
 """
+
+# TODO [ ] Get rid of all this deferToThread mess, or at least contain
+#          all of it into its own threadpool.
+
 import logging
 import os
 import socket
 import time
 
 from functools import partial
-from threading import Condition
 
 from twisted.internet import threads, defer
 from twisted.python import log
@@ -59,9 +62,8 @@ from leap.bitmask.util.privilege_policies import LinuxPolicyChecker
 from leap.common import certs as leap_certs
 
 from leap.keymanager import openpgp
-from leap.keymanager.errors import KeyAddressMismatch, KeyFingerprintMismatch
 
-from leap.soledad.client import NoStorageSecret, PassphraseTooShort
+from leap.soledad.client.secrets import NoStorageSecret, PassphraseTooShort
 
 logger = logging.getLogger(__name__)
 
@@ -906,52 +908,6 @@ class Keymanager(object):
         # NOTE: This feature is disabled right now since is dangerous
         return
 
-        new_key = ''
-        signal = None
-        try:
-            with open(filename, 'r') as keys_file:
-                new_key = keys_file.read()
-        except IOError as e:
-            logger.error("IOError importing key. {0!r}".format(e))
-            signal = self._signaler.keymanager_import_ioerror
-            self._signaler.signal(signal)
-            return
-
-        keymanager = self._keymanager_proxy
-        try:
-            # NOTE: parse_openpgp_ascii_key is not in keymanager anymore
-            #       the API for that will need some thinking 
-            public_key, private_key = keymanager.parse_openpgp_ascii_key(
-                new_key)
-        except (KeyAddressMismatch, KeyFingerprintMismatch) as e:
-            logger.error(repr(e))
-            signal = self._signaler.keymanager_import_datamismatch
-            self._signaler.signal(signal)
-            return
-
-        if public_key is None or private_key is None:
-            signal = self._signaler.keymanager_import_missingkey
-            self._signaler.signal(signal)
-            return
-
-        current_public_key = keymanager.get_key(username, openpgp.OpenPGPKey)
-        if public_key.address != current_public_key.address:
-            logger.error("The key does not match the ID")
-            signal = self._signaler.keymanager_import_addressmismatch
-            self._signaler.signal(signal)
-            return
-
-        keymanager.delete_key(self._key)
-        keymanager.delete_key(self._key_priv)
-        keymanager.put_key(public_key)
-        keymanager.put_key(private_key)
-        keymanager.send_key(openpgp.OpenPGPKey)
-
-        logger.debug('Import ok')
-        signal = self._signaler.keymanager_import_ok
-
-        self._signaler.signal(signal)
-
     def export_keys(self, username, filename):
         """
         Export the given username's keys to a file.
@@ -963,35 +919,50 @@ class Keymanager(object):
         """
         keymanager = self._keymanager_proxy
 
-        public_key = keymanager.get_key(username, openpgp.OpenPGPKey)
-        private_key = keymanager.get_key(username, openpgp.OpenPGPKey,
-                                         private=True)
-        try:
+        def export(keys):
+            public_key, private_key = keys
+            # XXX: This is blocking. We could use writeToFD, but is POSIX only
+            #      https://twistedmatrix.com/documents/current/api/twisted.internet.fdesc.html#writeToFD
             with open(filename, 'w') as keys_file:
                 keys_file.write(public_key.key_data)
                 keys_file.write(private_key.key_data)
 
             logger.debug('Export ok')
             self._signaler.signal(self._signaler.keymanager_export_ok)
-        except IOError as e:
-            logger.error("IOError exporting key. {0!r}".format(e))
+
+        def log_error(failure):
+            logger.error(
+                "Error exporting key. {0!r}".format(failure.value))
             self._signaler.signal(self._signaler.keymanager_export_error)
 
+        dpub = keymanager.get_key(username, openpgp.OpenPGPKey)
+        dpriv = keymanager.get_key(username, openpgp.OpenPGPKey,
+                                   private=True)
+        d = defer.gatherResults([dpub, dpriv])
+        d.addCallback(export)
+        d.addErrback(log_error)
+
     def list_keys(self):
         """
         List all the keys stored in the local DB.
         """
-        keys = self._keymanager_proxy.get_all_keys()
-        self._signaler.signal(self._signaler.keymanager_keys_list, keys)
+        d = self._keymanager_proxy.get_all_keys()
+        d.addCallback(
+            lambda keys:
+            self._signaler.signal(self._signaler.keymanager_keys_list, keys))
 
     def get_key_details(self, username):
         """
         List all the keys stored in the local DB.
         """
-        public_key = self._keymanager_proxy.get_key(username,
-                                                    openpgp.OpenPGPKey)
-        details = (public_key.key_id, public_key.fingerprint)
-        self._signaler.signal(self._signaler.keymanager_key_details, details)
+        def signal_details(public_key):
+            details = (public_key.key_id, public_key.fingerprint)
+            self._signaler.signal(self._signaler.keymanager_key_details,
+                                  details)
+
+        d = self._keymanager_proxy.get_key(username,
+                                           openpgp.OpenPGPKey)
+        d.addCallback(signal_details)
 
 
 class Mail(object):
@@ -1070,12 +1041,10 @@ class Mail(object):
         """
         Stop imap and wait until the service is stopped to signal that is done.
         """
-        cv = Condition()
-        cv.acquire()
-        threads.deferToThread(self._imap_controller.stop_imap_service, cv)
+        # FIXME just get a fucking deferred and signal as a callback, with
+        # timeout and cancellability
+        threads.deferToThread(self._imap_controller.stop_imap_service)
         logger.debug('Waiting for imap service to stop.')
-        cv.wait(self.SERVICE_STOP_TIMEOUT)
-        logger.debug('IMAP stopped')
         self._signaler.signal(self._signaler.imap_stopped)
 
     def stop_imap_service(self):
diff --git a/src/leap/bitmask/gui/mainwindow.py b/src/leap/bitmask/gui/mainwindow.py
index f5983abb..bf61a6ba 100644
--- a/src/leap/bitmask/gui/mainwindow.py
+++ b/src/leap/bitmask/gui/mainwindow.py
@@ -1191,9 +1191,13 @@ class MainWindow(QtGui.QMainWindow, SignalTracker):
         self._disconnect_login_wait()
 
         user = self._login_widget.get_logged_user()
-        domain = self._providers.get_selected_provider()
-        full_user_id = make_address(user, domain)
-        self._mail_conductor.userid = full_user_id
+        # XXX the widget now gives us the full user id.
+        # this is confusing.
+        #domain = self._providers.get_selected_provider()
+        #full_user_id = make_address(user, domain)
+        # XXX the casting to str (needed by smtp gateway) should be done
+        # in a better place.
+        self._mail_conductor.userid = str(user)
         self._start_eip_bootstrap()
         self.ui.action_create_new_account.setEnabled(True)
 
diff --git a/src/leap/bitmask/services/mail/conductor.py b/src/leap/bitmask/services/mail/conductor.py
index 0fb9f4fa..42bdd032 100644
--- a/src/leap/bitmask/services/mail/conductor.py
+++ b/src/leap/bitmask/services/mail/conductor.py
@@ -262,7 +262,7 @@ class MailConductor(IMAPControl, SMTPControl):
         if self._firewall is not None:
             self._firewall.start()
         if not offline:
-            logger.debug("not starting smtp in offline mode")
+            logger.debug("Starting smtp service...")
             self.start_smtp_service(download_if_needed=download_if_needed)
         self.start_imap_service()
 
diff --git a/src/leap/bitmask/services/mail/imap.py b/src/leap/bitmask/services/mail/imap.py
index 5db18cb9..d044a600 100644
--- a/src/leap/bitmask/services/mail/imap.py
+++ b/src/leap/bitmask/services/mail/imap.py
@@ -21,7 +21,9 @@ import logging
 import os
 import sys
 
+from leap.mail.constants import INBOX_NAME
 from leap.mail.imap.service import imap
+from leap.mail.incoming.service import IncomingMail, INCOMING_CHECK_PERIOD
 from twisted.python import log
 
 logger = logging.getLogger(__name__)
@@ -49,6 +51,9 @@ def get_mail_check_period():
         logger.warning("Unhandled error while getting %s: %r" % (
             INCOMING_CHECK_PERIOD_ENV,
             exc))
+
+    if period is None:
+        period = INCOMING_CHECK_PERIOD
     return period
 
 
@@ -61,12 +66,39 @@ def start_imap_service(*args, **kwargs):
     from leap.bitmask.config import flags
     logger.debug('Launching imap service')
 
-    override_period = get_mail_check_period()
-    if override_period:
-        kwargs['check_period'] = override_period
-
     if flags.MAIL_LOGFILE:
         log.startLogging(open(flags.MAIL_LOGFILE, 'w'))
         log.startLogging(sys.stdout)
 
     return imap.run_service(*args, **kwargs)
+
+
+def start_incoming_mail_service(keymanager, soledad, imap_factory, userid):
+    """
+    Initalizes and starts the incomming mail service.
+
+    :returns: a Deferred that will be fired with the IncomingMail instance
+    """
+    def setUpIncomingMail(inbox):
+        incoming_mail = IncomingMail(
+            keymanager,
+            soledad,
+            inbox,
+            userid,
+            check_period=get_mail_check_period())
+        return incoming_mail
+
+    # XXX: do I really need to know here how to get a mailbox??
+    # XXX: ideally, the parent service in mail would take care of initializing
+    # the account, and passing the mailbox to the incoming service.
+    # In an even better world, we just would subscribe to a channel that would
+    # pass us the serialized object to be inserted.
+    # XXX: I think we might be at risk here because the account top object
+    # currently just returns as many mailbox objects as it's asked for, so
+    # we should be careful about concurrent operations (ie, maybe just use
+    # this one to do inserts, or let account have an in-memory map of
+    # mailboxes, and just return references to them).
+    acc = imap_factory.theAccount
+    d = acc.callWhenReady(lambda _: acc.getMailbox(INBOX_NAME))
+    d.addCallback(setUpIncomingMail)
+    return d
diff --git a/src/leap/bitmask/services/mail/imapcontroller.py b/src/leap/bitmask/services/mail/imapcontroller.py
index d0bf4c34..d374ac29 100644
--- a/src/leap/bitmask/services/mail/imapcontroller.py
+++ b/src/leap/bitmask/services/mail/imapcontroller.py
@@ -43,9 +43,12 @@ class IMAPController(object):
         self._soledad = soledad
         self._keymanager = keymanager
 
-        self.imap_service = None
+        # XXX: this should live in its own controller
+        # or, better, just be managed by a composite Mail Service in
+        # leap.mail.
         self.imap_port = None
         self.imap_factory = None
+        self.incoming_mail_service = None
 
     def start_imap_service(self, userid, offline=False):
         """
@@ -58,46 +61,53 @@ class IMAPController(object):
         """
         logger.debug('Starting imap service')
 
-        self.imap_service, self.imap_port, \
-            self.imap_factory = imap.start_imap_service(
-                self._soledad,
-                self._keymanager,
-                userid=userid,
-                offline=offline)
+        self.imap_port, self.imap_factory = imap.start_imap_service(
+            self._soledad,
+            userid=userid)
+
+        def start_incoming_service(incoming_mail):
+            d = incoming_mail.startService()
+            d.addCallback(lambda started: incoming_mail)
+            return d
+
+        def assign_incoming_service(incoming_mail):
+            self.incoming_mail_service = incoming_mail
+            return incoming_mail
 
         if offline is False:
-            logger.debug("Starting loop")
-            self.imap_service.start_loop()
+            d = imap.start_incoming_mail_service(
+                self._keymanager,
+                self._soledad,
+                self.imap_factory,
+                userid)
+            d.addCallback(start_incoming_service)
+            d.addCallback(assign_incoming_service)
+            d.addErrback(lambda f: logger.error(f.printTraceback()))
 
-    def stop_imap_service(self, cv):
+    def stop_imap_service(self):
         """
         Stop IMAP service (fetcher, factory and port).
-
-        :param cv: A condition variable to which we can signal when imap
-                   indeed stops.
-        :type cv: threading.Condition
         """
-        if self.imap_service is not None:
+        if self.incoming_mail_service is not None:
             # Stop the loop call in the fetcher
-            self.imap_service.stop()
-            self.imap_service = None
 
+            # XXX BUG -- the deletion of the reference should be made
+            # after stopService() triggers its deferred (ie, cleanup has been
+            # made)
+            self.incoming_mail_service.stopService()
+            self.incoming_mail_service = None
+
+        if self.imap_port is not None:
             # Stop listening on the IMAP port
             self.imap_port.stopListening()
 
             # Stop the protocol
-            self.imap_factory.theAccount.closed = True
-            self.imap_factory.doStop(cv)
-        else:
-            # Release the condition variable so the caller doesn't have to wait
-            cv.acquire()
-            cv.notify()
-            cv.release()
+            self.imap_factory.doStop()
 
     def fetch_incoming_mail(self):
         """
         Fetch incoming mail.
         """
-        if self.imap_service:
+        if self.incoming_mail_service is not None:
             logger.debug('Client connected, fetching mail...')
-            self.imap_service.fetch()
+            self.incoming_mail_service.fetch()
diff --git a/src/leap/bitmask/services/mail/plumber.py b/src/leap/bitmask/services/mail/plumber.py
index 1af65c5d..58625db5 100644
--- a/src/leap/bitmask/services/mail/plumber.py
+++ b/src/leap/bitmask/services/mail/plumber.py
@@ -17,6 +17,8 @@
 """
 Utils for manipulating local mailboxes.
 """
+# TODO --- this module has not yet catched up with 0.9.0
+
 import getpass
 import logging
 import os
@@ -32,9 +34,7 @@ from leap.bitmask.provider import get_provider_path
 from leap.bitmask.services.soledad.soledadbootstrapper import get_db_paths
 from leap.bitmask.util import flatten, get_path_prefix
 
-from leap.mail.imap.account import SoledadBackedAccount
-from leap.mail.imap.memorystore import MemoryStore
-from leap.mail.imap.soledadstore import SoledadStore
+from leap.mail.imap.account import IMAPAccount
 from leap.soledad.client import Soledad
 
 logger = logging.getLogger(__name__)
@@ -140,11 +140,7 @@ class MBOXPlumber(object):
         self.sol = initialize_soledad(
             self.uuid, self.userid, self.passwd,
             secrets, localdb, "/tmp", "/tmp")
-        memstore = MemoryStore(
-            permanent_store=SoledadStore(self.sol),
-            write_period=5)
-        self.acct = SoledadBackedAccount(self.userid, self.sol,
-                                         memstore=memstore)
+        self.acct = IMAPAccount(self.userid, self.sol)
         return True
 
     #
diff --git a/src/leap/bitmask/services/mail/smtpbootstrapper.py b/src/leap/bitmask/services/mail/smtpbootstrapper.py
index 9dd61488..6f45469b 100644
--- a/src/leap/bitmask/services/mail/smtpbootstrapper.py
+++ b/src/leap/bitmask/services/mail/smtpbootstrapper.py
@@ -120,6 +120,7 @@ class SMTPBootstrapper(AbstractBootstrapper):
             self._provider_config, about_to_download=True)
 
         from leap.mail.smtp import setup_smtp_gateway
+
         self._smtp_service, self._smtp_port = setup_smtp_gateway(
             port=2013,
             userid=self._userid,
@@ -152,7 +153,7 @@ class SMTPBootstrapper(AbstractBootstrapper):
         self._provider_config = ProviderConfig.get_provider_config(domain)
         self._keymanager = keymanager
         self._smtp_config = SMTPConfig()
-        self._userid = userid
+        self._userid = str(userid)
         self._download_if_needed = download_if_needed
 
         try:
diff --git a/src/leap/bitmask/services/soledad/soledadbootstrapper.py b/src/leap/bitmask/services/soledad/soledadbootstrapper.py
index 2044a27c..045b2e19 100644
--- a/src/leap/bitmask/services/soledad/soledadbootstrapper.py
+++ b/src/leap/bitmask/services/soledad/soledadbootstrapper.py
@@ -27,7 +27,7 @@ from ssl import SSLError
 from sqlite3 import ProgrammingError as sqlite_ProgrammingError
 
 from u1db import errors as u1db_errors
-from twisted.internet import threads
+from twisted.internet import threads, defer
 from zope.proxy import sameProxiedObjects
 from pysqlcipher.dbapi2 import ProgrammingError as sqlcipher_ProgrammingError
 
@@ -45,7 +45,8 @@ from leap.common.files import which
 from leap.keymanager import KeyManager, openpgp
 from leap.keymanager.errors import KeyNotFound
 from leap.soledad.common.errors import InvalidAuthTokenError
-from leap.soledad.client import Soledad, BootstrapSequenceError
+from leap.soledad.client import Soledad
+from leap.soledad.client.secrets import BootstrapSequenceError
 
 logger = logging.getLogger(__name__)
 
@@ -317,19 +318,26 @@ class SoledadBootstrapper(AbstractBootstrapper):
         if flags.OFFLINE:
             self._init_keymanager(self._address, token)
         else:
-            try:
-                address = make_address(
-                    self._user, self._provider_config.get_domain())
-                self._init_keymanager(address, token)
-                self._keymanager.get_key(
+            address = make_address(
+                self._user, self._provider_config.get_domain())
+
+            def key_not_found(failure):
+                if failure.check(KeyNotFound):
+                    logger.debug("Key not found. Generating key for %s" %
+                                 (address,))
+                    self._do_soledad_sync()
+                else:
+                    return failure
+
+            d = self._init_keymanager(address, token)
+            d.addCallback(
+                lambda _: self._keymanager.get_key(
                     address, openpgp.OpenPGPKey,
-                    private=True, fetch_remote=False)
-                d = threads.deferToThread(self._do_soledad_sync)
-                d.addErrback(self._soledad_sync_errback)
-            except KeyNotFound:
-                logger.debug("Key not found. Generating key for %s" %
-                             (address,))
-                self._do_soledad_sync()
+                    private=True, fetch_remote=False))
+            d.addCallbacks(
+                lambda _: threads.deferToThread(self._do_soledad_sync),
+                key_not_found)
+            d.addErrback(self._soledad_sync_errback)
 
     def _pick_server(self, uuid):
         """
@@ -537,6 +545,7 @@ class SoledadBootstrapper(AbstractBootstrapper):
         :type address: str
         :param token: the auth token for accessing webapp.
         :type token: str
+        :rtype: Deferred
         """
         srp_auth = self.srpauth
         logger.debug('initializing keymanager...')
@@ -576,20 +585,27 @@ class SoledadBootstrapper(AbstractBootstrapper):
         if flags.OFFLINE is False:
             # make sure key is in server
             logger.debug('Trying to send key to server...')
-            try:
-                self._keymanager.send_key(openpgp.OpenPGPKey)
-            except KeyNotFound:
-                logger.debug('No key found for %s, will generate soon.'
-                             % address)
-            except Exception as exc:
-                logger.error("Error sending key to server.")
-                logger.exception(exc)
-                # but we do not raise
+
+            def send_errback(failure):
+                if failure.check(KeyNotFound):
+                    logger.debug('No key found for %s, will generate soon.'
+                                 % address)
+                else:
+                    logger.error("Error sending key to server.")
+                    logger.exception(failure.value)
+                    # but we do not raise
+
+            d = self._keymanager.send_key(openpgp.OpenPGPKey)
+            d.addErrback(send_errback)
+            return d
+        else:
+            return defer.succeed(None)
 
     def _gen_key(self):
         """
         Generates the key pair if needed, uploads it to the webapp and
         nickserver
+        :rtype: Deferred
         """
         leap_assert(self._provider_config is not None,
                     "We need a provider configuration!")
@@ -600,30 +616,33 @@ class SoledadBootstrapper(AbstractBootstrapper):
             self._user, self._provider_config.get_domain())
         logger.debug("Retrieving key for %s" % (address,))
 
-        try:
-            self._keymanager.get_key(
-                address, openpgp.OpenPGPKey, private=True, fetch_remote=False)
-            return
-        except KeyNotFound:
-            logger.debug("Key not found. Generating key for %s" % (address,))
-
-        # generate key
-        try:
-            self._keymanager.gen_key(openpgp.OpenPGPKey)
-        except Exception as exc:
-            logger.error("Error while generating key!")
-            logger.exception(exc)
-            raise
-
-        # send key
-        try:
-            self._keymanager.send_key(openpgp.OpenPGPKey)
-        except Exception as exc:
-            logger.error("Error while sending key!")
-            logger.exception(exc)
-            raise
-
-        logger.debug("Key generated successfully.")
+        def if_not_found_generate(failure):
+            if failure.check(KeyNotFound):
+                logger.debug("Key not found. Generating key for %s"
+                             % (address,))
+                d = self._keymanager.gen_key(openpgp.OpenPGPKey)
+                d.addCallbacks(send_key, log_key_error("generating"))
+                return d
+            else:
+                return failure
+
+        def send_key(_):
+            d = self._keymanager.send_key(openpgp.OpenPGPKey)
+            d.addCallbacks(
+                lambda _: logger.debug("Key generated successfully."),
+                log_key_error("sending"))
+
+        def log_key_error(step):
+            def log_err(failure):
+                logger.error("Error while %s key!", (step,))
+                logger.exception(failure.value)
+                return failure
+            return log_err
+
+        d = self._keymanager.get_key(
+            address, openpgp.OpenPGPKey, private=True, fetch_remote=False)
+        d.addErrback(if_not_found_generate)
+        return d
 
     def run_soledad_setup_checks(self, provider_config, user, password,
                                  download_if_needed=False):
@@ -664,9 +683,10 @@ class SoledadBootstrapper(AbstractBootstrapper):
             self.load_and_sync_soledad(uuid)
 
             if not flags.OFFLINE:
-                self._gen_key()
-
-            self._signaler.signal(signal_finished)
+                d = self._gen_key()
+                d.addCallback(lambda _: self._signaler.signal(signal_finished))
+            else:
+                self._signaler.signal(signal_finished)
         except Exception as e:
             # TODO: we should handle more specific exceptions in here
             self._soledad = None