Merge tag '0.4.0rc2' into debian/0.4.0rc2

Tag leap.mail version 0.4.0rc2

2 files changed, 61 insertions, 136 deletions

diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac
index 651f71b..2045757 100644
--- a/src/leap/mail/imap/service/imap-server.tac
+++ b/src/leap/mail/imap/service/imap-server.tac
@@ -23,9 +23,9 @@ For now, and for debugging/testing purposes, you need
 to pass a config file with the following structure:
 
 [leap_mail]
-userid = "user@provider"
-uuid = "deadbeefdeadabad"
-passwd = "supersecret" # optional, will get prompted if not found.
+userid = 'user@provider'
+uuid = 'deadbeefdeadabad'
+passwd = 'supersecret' # optional, will get prompted if not found.
 """
 import ConfigParser
 import getpass
@@ -53,38 +53,17 @@ def initialize_soledad(uuid, email, passwd,
     :param tempdir: path to temporal dir
     :rtype: Soledad instance
     """
-    # XXX TODO unify with an authoritative source of mocks
-    # for soledad (or partial initializations).
-    # This is copied from the imap tests.
-
     server_url = "http://provider"
     cert_file = ""
 
-    class Mock(object):
-        def __init__(self, return_value=None):
-            self._return = return_value
-
-        def __call__(self, *args, **kwargs):
-            return self._return
-
-    class MockSharedDB(object):
-
-        get_doc = Mock()
-        put_doc = Mock()
-        lock = Mock(return_value=('atoken', 300))
-        unlock = Mock(return_value=True)
-
-        def __call__(self):
-            return self
-
-    Soledad._shared_db = MockSharedDB()
     soledad = Soledad(
         uuid,
         passwd,
         secrets,
         localdb,
         server_url,
-        cert_file)
+        cert_file,
+        syncable=False)
 
     return soledad
 
@@ -95,9 +74,9 @@ def initialize_soledad(uuid, email, passwd,
 print "[+] Running LEAP IMAP Service"
 
 
-bmconf = os.environ.get("LEAP_MAIL_CONF", "")
+bmconf = os.environ.get("LEAP_MAIL_CONFIG", "")
 if not bmconf:
-    print ("[-] Please set LEAP_MAIL_CONF environment variable "
+    print ("[-] Please set LEAP_MAIL_CONFIG environment variable "
            "pointing to your config.")
     sys.exit(1)
 
@@ -131,6 +110,7 @@ tempdir = "/tmp/"
 
 # Ad-hoc soledad/keymanager initialization.
 
+print "[~] user:", userid
 soledad = initialize_soledad(uuid, userid, passwd, secrets,
                              localdb, gnupg_home, tempdir)
 km_args = (userid, "https://localhost", soledad)
diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py
index 10ba32a..c3ae59a 100644
--- a/src/leap/mail/imap/service/imap.py
+++ b/src/leap/mail/imap/service/imap.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # imap.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013-2015 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -15,58 +15,27 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
-Imap service initialization
+IMAP service initialization
 """
 import logging
 import os
-import time
 
-from twisted.internet import defer, threads
-from twisted.internet.protocol import ServerFactory
+from collections import defaultdict
+
+from twisted.internet import reactor
 from twisted.internet.error import CannotListenError
+from twisted.internet.protocol import ServerFactory
 from twisted.mail import imap4
 from twisted.python import log
 
-logger = logging.getLogger(__name__)
+from leap.common.events import emit, catalog
+from leap.common.check import leap_check
+from leap.mail.imap.account import IMAPAccount
+from leap.mail.imap.server import LEAPIMAPServer
 
-from leap.common import events as leap_events
-from leap.common.check import leap_assert, leap_assert_type, leap_check
-from leap.keymanager import KeyManager
-from leap.mail.imap.account import SoledadBackedAccount
-from leap.mail.imap.fetch import LeapIncomingMail
-from leap.mail.imap.memorystore import MemoryStore
-from leap.mail.imap.server import LeapIMAPServer
-from leap.mail.imap.soledadstore import SoledadStore
-from leap.soledad.client import Soledad
-
-# The default port in which imap service will run
-IMAP_PORT = 1984
+# TODO: leave only an implementor of IService in here
 
-# The period between succesive checks of the incoming mail
-# queue (in seconds)
-INCOMING_CHECK_PERIOD = 60
-
-from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED
-from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START
-
-######################################################
-# Temporary workaround for RecursionLimit when using
-# qt4reactor. Do remove when we move to poll or select
-# reactor, which do not show those problems. See #4974
-import resource
-import sys
-
-try:
-    sys.setrecursionlimit(10**7)
-except Exception:
-    print "Error setting recursion limit"
-try:
-    # Increase max stack size from 8MB to 256MB
-    resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1))
-except Exception:
-    print "Error setting stack size"
-
-######################################################
+logger = logging.getLogger(__name__)
 
 DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None)
 if DO_MANHOLE:
@@ -81,6 +50,9 @@ if DO_PROFILE:
     pr = cProfile.Profile()
     pr.enable()
 
+# The default port in which imap service will run
+IMAP_PORT = 1984
+
 
 class IMAPAuthRealm(object):
     """
@@ -97,6 +69,7 @@ class LeapIMAPFactory(ServerFactory):
     Factory for a IMAP4 server with soledad remote sync and gpg-decryption
     capabilities.
     """
+    protocol = LEAPIMAPServer
 
     def __init__(self, uuid, userid, soledad):
         """
@@ -114,14 +87,10 @@ class LeapIMAPFactory(ServerFactory):
         self._uuid = uuid
         self._userid = userid
         self._soledad = soledad
-        self._memstore = MemoryStore(
-            permanent_store=SoledadStore(soledad))
 
-        theAccount = SoledadBackedAccount(
-            uuid, soledad=soledad,
-            memstore=self._memstore)
+        theAccount = IMAPAccount(uuid, soledad)
         self.theAccount = theAccount
-
+        self._connections = defaultdict()
         # XXX how to pass the store along?
 
     def buildProtocol(self, addr):
@@ -131,91 +100,66 @@ class LeapIMAPFactory(ServerFactory):
         :param addr: remote ip address
         :type addr:  str
         """
-        imapProtocol = LeapIMAPServer(
+        # TODO should reject anything from addr != localhost,
+        # just in case.
+        log.msg("Building protocol for connection %s" % addr)
+        imapProtocol = self.protocol(
             uuid=self._uuid,
             userid=self._userid,
             soledad=self._soledad)
         imapProtocol.theAccount = self.theAccount
         imapProtocol.factory = self
+
+        self._connections[addr] = imapProtocol
         return imapProtocol
 
-    def doStop(self, cv=None):
+    def stopFactory(self):
+        # say bye!
+        for conn, proto in self._connections.items():
+            log.msg("Closing connections for %s" % conn)
+            proto.close_server_connection()
+
+    def doStop(self):
         """
         Stops imap service (fetcher, factory and port).
-
-        :param cv: A condition variable to which we can signal when imap
-                   indeed stops.
-        :type cv: threading.Condition
-        :return: a Deferred that stops and flushes the in memory store data to
-                 disk in another thread.
-        :rtype: Deferred
         """
+        # mark account as unusable, so any imap command will fail
+        # with unauth state.
+        self.theAccount.end_session()
+
+        # TODO should wait for all the pending deferreds,
+        # the twisted way!
         if DO_PROFILE:
             log.msg("Stopping PROFILING")
             pr.disable()
             pr.dump_stats(PROFILE_DAT)
 
-        ServerFactory.doStop(self)
-
-        if cv is not None:
-            def _stop_imap_cb():
-                logger.debug('Stopping in memory store.')
-                self._memstore.stop_and_flush()
-                while not self._memstore.producer.is_queue_empty():
-                    logger.debug('Waiting for queue to be empty.')
-                    # TODO use a gatherResults over the new/dirty
-                    # deferred list,
-                    # as in memorystore's expunge() method.
-                    time.sleep(1)
-                # notify that service has stopped
-                logger.debug('Notifying that service has stopped.')
-                cv.acquire()
-                cv.notify()
-                cv.release()
-
-            return threads.deferToThread(_stop_imap_cb)
+        return ServerFactory.doStop(self)
 
 
-def run_service(*args, **kwargs):
+def run_service(store, **kwargs):
     """
     Main entry point to run the service from the client.
 
-    :returns: the LoopingCall instance that will have to be stoppped
-              before shutting down the client, the port as returned by
-              the reactor when starts listening, and the factory for
-              the protocol.
-    """
-    from twisted.internet import reactor
-    # it looks like qtreactor does not honor this,
-    # but other reactors should.
-    reactor.suggestThreadPoolSize(20)
+    :param store: a soledad instance
 
-    leap_assert(len(args) == 2)
-    soledad, keymanager = args
-    leap_assert_type(soledad, Soledad)
-    leap_assert_type(keymanager, KeyManager)
+    :returns: the port as returned by the reactor when starts listening, and
+              the factory for the protocol.
+    """
+    leap_check(store, "store cannot be None")
+    # XXX this can also be a ProxiedObject, FIXME
+    # leap_assert_type(store, Soledad)
 
     port = kwargs.get('port', IMAP_PORT)
-    check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD)
     userid = kwargs.get('userid', None)
     leap_check(userid is not None, "need an user id")
-    offline = kwargs.get('offline', False)
 
-    uuid = soledad._get_uuid()
-    factory = LeapIMAPFactory(uuid, userid, soledad)
+    uuid = store.uuid
+    factory = LeapIMAPFactory(uuid, userid, store)
 
     try:
         tport = reactor.listenTCP(port, factory,
                                   interface="localhost")
-        if not offline:
-            fetcher = LeapIncomingMail(
-                keymanager,
-                soledad,
-                factory.theAccount,
-                check_period,
-                userid)
-        else:
-            fetcher = None
     except CannotListenError:
         logger.error("IMAP Service failed to start: "
                      "cannot listen in port %s" % (port,))
@@ -223,7 +167,6 @@ def run_service(*args, **kwargs):
         logger.error("Error launching IMAP service: %r" % (exc,))
     else:
         # all good.
-        # (the caller has still to call fetcher.start_loop)
 
         if DO_MANHOLE:
             # TODO get pass from env var.too.
@@ -235,8 +178,10 @@ def run_service(*args, **kwargs):
             reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory,
                               interface="127.0.0.1")
         logger.debug("IMAP4 Server is RUNNING in port  %s" % (port,))
-        leap_events.signal(IMAP_SERVICE_STARTED, str(port))
-        return fetcher, tport, factory
+        emit(catalog.IMAP_SERVICE_STARTED, str(port))
+
+        # FIXME -- change service signature
+        return tport, factory
 
     # not ok, signal error.
-    leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port))
+    emit(catalog.IMAP_SERVICE_FAILED_TO_START, str(port))