Merge remote-tracking branch 'kali/feature/refactor_imap_fetch' into develop

5 files changed, 100 insertions, 61 deletions

diff --git a/mail/changes/feature_cert_auth b/mail/changes/feature_cert_auth
new file mode 100644
index 00000000..23cdf909
--- /dev/null
+++ b/mail/changes/feature_cert_auth
@@ -0,0 +1 @@
+  o Add client certificate authentication. Closes #3376.
\ No newline at end of file
diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py
index 48a45e68..1c418130 100644
--- a/mail/src/leap/mail/imap/fetch.py
+++ b/mail/src/leap/mail/imap/fetch.py
@@ -1,15 +1,16 @@
 import logging
 import json
+import ssl
 
 from twisted.python import log
 from twisted.internet import defer
+from twisted.internet.task import LoopingCall
 from twisted.internet.threads import deferToThread
 
 from leap.common.check import leap_assert, leap_assert_type
+from leap.keymanager import openpgp
 from leap.soledad import Soledad
 
-from leap.common.keymanager import openpgp
-
 logger = logging.getLogger(__name__)
 
 
@@ -26,7 +27,8 @@ class LeapIncomingMail(object):
     INCOMING_KEY = "incoming"
     CONTENT_KEY = "content"
 
-    def __init__(self, keymanager, soledad, imap_account):
+    def __init__(self, keymanager, soledad, imap_account,
+                 check_period):
 
         """
         Initialize LeapIMAP.
@@ -39,10 +41,15 @@ class LeapIncomingMail(object):
 
         :param imap_account: the account to fetch periodically
         :type imap_account: SoledadBackedAccount
+
+        :param check_period: the period to fetch new mail, in seconds.
+        :type check_period: int
         """
 
         leap_assert(keymanager, "need a keymanager to initialize")
         leap_assert_type(soledad, Soledad)
+        leap_assert(check_period, "need a period to check incoming mail")
+        leap_assert_type(check_period, int)
 
         self._keymanager = keymanager
         self._soledad = soledad
@@ -51,6 +58,16 @@ class LeapIncomingMail(object):
 
         self._pkey = self._keymanager.get_all_keys_in_local_db(
             private=True).pop()
+        self._loop = None
+        self._check_period = check_period
+
+        self._create_soledad_indexes()
+
+    def _create_soledad_indexes(self):
+        """
+        Create needed indexes on soledad.
+        """
+        self._soledad.create_index("just-mail", "incoming")
 
     def fetch(self):
         """
@@ -64,16 +81,33 @@ class LeapIncomingMail(object):
         d.addCallbacks(self._process_doclist, self._sync_soledad_err)
         return d
 
+    def start_loop(self):
+        """
+        Starts a loop to fetch mail.
+        """
+        self._loop = LoopingCall(self.fetch)
+        self._loop.start(self._check_period)
+
+    def stop(self):
+        """
+        Stops the loop that fetches mail.
+        """
+        if self._loop:
+            self._loop.stop()
+
     def _sync_soledad(self):
         log.msg('syncing soledad...')
         logger.debug('in soledad sync')
-        #import ipdb; ipdb.set_trace()
 
-        self._soledad.sync()
-        gen, doclist = self._soledad.get_all_docs()
-        #logger.debug("there are %s docs" % (len(doclist),))
-        log.msg("there are %s docs" % (len(doclist),))
-        return doclist
+        try:
+            self._soledad.sync()
+            doclist = self._soledad.get_from_index("just-mail", "*")
+            #log.msg("there are %s mails" % (len(doclist),))
+            return doclist
+        except ssl.SSLError as exc:
+            logger.warning('SSL Error while syncing soledad: %r' % (exc,))
+        except Exception as exc:
+            logger.warning('Error while syncing soledad: %r' % (exc,))
 
     def _sync_soledad_err(self, f):
         log.err("error syncing soledad: %s" % (f.value,))
@@ -81,6 +115,8 @@ class LeapIncomingMail(object):
 
     def _process_doclist(self, doclist):
         log.msg('processing doclist')
+        if not doclist:
+            return
         for doc in doclist:
             keys = doc.content.keys()
             if self.ENC_SCHEME_KEY in keys and self.ENC_JSON_KEY in keys:
diff --git a/mail/src/leap/mail/imap/service/imap.py b/mail/src/leap/mail/imap/service/imap.py
index 49d54e3e..9e331b69 100644
--- a/mail/src/leap/mail/imap/service/imap.py
+++ b/mail/src/leap/mail/imap/service/imap.py
@@ -20,15 +20,13 @@ Imap service initialization
 import logging
 logger = logging.getLogger(__name__)
 
-#from twisted.application import internet, service
 from twisted.internet.protocol import ServerFactory
-from twisted.internet.task import LoopingCall
 
 from twisted.mail import imap4
 from twisted.python import log
 
 from leap.common.check import leap_assert, leap_assert_type
-from leap.common.keymanager import KeyManager
+from leap.keymanager import KeyManager
 from leap.mail.imap.server import SoledadBackedAccount
 from leap.mail.imap.fetch import LeapIncomingMail
 from leap.soledad import Soledad
@@ -36,8 +34,8 @@ from leap.soledad import Soledad
 IMAP_PORT = 9930
 # The default port in which imap service will run
 
-#INCOMING_CHECK_PERIOD = 10
-INCOMING_CHECK_PERIOD = 5
+# INCOMING_CHECK_PERIOD = 5
+INCOMING_CHECK_PERIOD = 60
 # The period between succesive checks of the incoming mail
 # queue (in seconds)
 
@@ -127,6 +125,9 @@ class LeapIMAPFactory(ServerFactory):
 def run_service(*args, **kwargs):
     """
     Main entry point to run the service from the client.
+
+    :returns: the LoopingCall instance that will have to be stoppped
+              before shutting down the client.
     """
     leap_assert(len(args) == 2)
     soledad, keymanager = args
@@ -139,29 +140,16 @@ def run_service(*args, **kwargs):
     uuid = soledad._get_uuid()
     factory = LeapIMAPFactory(uuid, soledad)
 
-    # ---- for application framework
-    #application = service.Application("LEAP IMAP4 Local Service")
-    #imapService = internet.TCPServer(port, factory)
-    #imapService.setServiceParent(application)
-
     from twisted.internet import reactor
     reactor.listenTCP(port, factory)
 
     fetcher = LeapIncomingMail(
         keymanager,
         soledad,
-        factory.theAccount)
+        factory.theAccount,
+        check_period)
 
-    lc = LoopingCall(fetcher.fetch)
-    lc.start(check_period)
+    fetcher.start_loop()
 
-    # ---- for application framework
-    #internet.TimerService(
-        #check_period,
-        #fetcher.fetch).setServiceParent(application)
-
-    logger.debug('----------------------------------------')
     logger.debug("IMAP4 Server is RUNNING in port  %s" % (port,))
-
-    #log.msg("IMAP4 Server is RUNNING in port  %s" % (port,))
-    #return application
+    return fetcher
diff --git a/mail/src/leap/mail/smtp/__init__.py b/mail/src/leap/mail/smtp/__init__.py
index 78eb4f88..3b4d9d66 100644
--- a/mail/src/leap/mail/smtp/__init__.py
+++ b/mail/src/leap/mail/smtp/__init__.py
@@ -25,8 +25,8 @@ from twisted.internet import reactor
 from leap.mail.smtp.smtprelay import SMTPFactory
 
 
-def setup_smtp_relay(port, keymanager, smtp_host, smtp_port, smtp_username,
-                     smtp_password, encrypted_only):
+def setup_smtp_relay(port, keymanager, smtp_host, smtp_port,
+                     smtp_cert, smtp_key, encrypted_only):
     """
     Setup SMTP relay to run with Twisted.
 
@@ -42,10 +42,10 @@ def setup_smtp_relay(port, keymanager, smtp_host, smtp_port, smtp_username,
     @type smtp_host: str
     @param smtp_port:  The port of the remote SMTP server.
     @type smtp_port: int
-    @param smtp_username: The username used to connect to remote SMTP server.
-    @type smtp_username: str
-    @param smtp_password: The password used to connect to remote SMTP server.
-    @type smtp_password: str
+    @param smtp_cert: The client certificate for authentication.
+    @type smtp_cert: str
+    @param smtp_key: The client key for authentication.
+    @type smtp_key: str
     @param encrypted_only: Whether the SMTP relay should send unencrypted mail
         or not.
     @type encrypted_only: bool
@@ -56,15 +56,15 @@ def setup_smtp_relay(port, keymanager, smtp_host, smtp_port, smtp_username,
     # {
     #     'host': '<host>',
     #     'port': <int>,
-    #     'username': '<username>',
-    #     'password': '<password>',
+    #     'cert': '<cert path>',
+    #     'key': '<key path>',
     #     'encrypted_only': <True/False>
     # }
     config = {
         'host': smtp_host,
         'port': smtp_port,
-        'username': smtp_username,
-        'password': smtp_password,
+        'cert': smtp_cert,
+        'key': smtp_key,
         'encrypted_only': encrypted_only
     }
 
diff --git a/mail/src/leap/mail/smtp/smtprelay.py b/mail/src/leap/mail/smtp/smtprelay.py
index 17388409..e5a56146 100644
--- a/mail/src/leap/mail/smtp/smtprelay.py
+++ b/mail/src/leap/mail/smtp/smtprelay.py
@@ -19,16 +19,12 @@
 LEAP SMTP encrypted relay.
 """
 
-import re
-import os
-import tempfile
-
-
 from zope.interface import implements
 from StringIO import StringIO
+from OpenSSL import SSL
 from twisted.mail import smtp
 from twisted.internet.protocol import ServerFactory
-from twisted.internet import reactor
+from twisted.internet import reactor, ssl
 from twisted.internet import defer
 from twisted.python import log
 from email.Header import Header
@@ -63,8 +59,8 @@ class MalformedConfig(Exception):
 
 HOST_KEY = 'host'
 PORT_KEY = 'port'
-USERNAME_KEY = 'username'
-PASSWORD_KEY = 'password'
+CERT_KEY = 'cert'
+KEY_KEY = 'key'
 ENCRYPTED_ONLY_KEY = 'encrypted_only'
 
 
@@ -89,17 +85,17 @@ def assert_config_structure(config):
     leap_assert_type(config[HOST_KEY], str)
     leap_assert(PORT_KEY in config)
     leap_assert_type(config[PORT_KEY], int)
-    leap_assert(USERNAME_KEY in config)
-    leap_assert_type(config[USERNAME_KEY], str)
-    leap_assert(PASSWORD_KEY in config)
-    leap_assert_type(config[PASSWORD_KEY], str)
+    leap_assert(CERT_KEY in config)
+    leap_assert_type(config[CERT_KEY], str)
+    leap_assert(KEY_KEY in config)
+    leap_assert_type(config[KEY_KEY], str)
     leap_assert(ENCRYPTED_ONLY_KEY in config)
     leap_assert_type(config[ENCRYPTED_ONLY_KEY], bool)
     # assert received params are not empty
     leap_assert(config[HOST_KEY] != '')
     leap_assert(config[PORT_KEY] is not 0)
-    leap_assert(config[USERNAME_KEY] != '')
-    leap_assert(config[PASSWORD_KEY] != '')
+    leap_assert(config[CERT_KEY] != '')
+    leap_assert(config[KEY_KEY] != '')
 
 
 def validate_address(address):
@@ -143,8 +139,8 @@ class SMTPFactory(ServerFactory):
                 {
                     HOST_KEY: '<str>',
                     PORT_KEY: <int>,
-                    USERNAME_KEY: '<str>',
-                    PASSWORD_KEY: '<str>',
+                    CERT_KEY: '<str>',
+                    KEY_KEY: '<str>',
                     ENCRYPTED_ONLY_KEY: <bool>,
                 }
         @type config: dict
@@ -294,6 +290,18 @@ class SMTPDelivery(object):
 # EncryptedMessage
 #
 
+class CtxFactory(ssl.ClientContextFactory):
+    def __init__(self, cert, key):
+        self.cert = cert
+        self.key = key
+
+    def getContext(self):
+        self.method = SSL.TLSv1_METHOD  #SSLv23_METHOD
+        ctx = ssl.ClientContextFactory.getContext(self)
+        ctx.use_certificate_file(self.cert)
+        ctx.use_privatekey_file(self.key)
+        return ctx
+
 class EncryptedMessage(object):
     """
     Receive plaintext from client, encrypt it and send message to a
@@ -405,17 +413,23 @@ class EncryptedMessage(object):
         @rtype: twisted.internet.defer.Deferred
         """
         msg = self._message.as_string(False)
+
+        log.msg("Connecting to SMTP server %s:%s" % (self._config[HOST_KEY],
+                                                     self._config[PORT_KEY]))
+
         d = defer.Deferred()
         factory = smtp.ESMTPSenderFactory(
-            self._config[USERNAME_KEY],
-            self._config[PASSWORD_KEY],
+            "",
+            "",
             self._fromAddress.addrstr,
             self._user.dest.addrstr,
             StringIO(msg),
             d,
-            requireAuthentication=False,  # for now do unauth, see issue #2474
+            contextFactory=CtxFactory(self._config[CERT_KEY],
+                                      self._config[KEY_KEY]),
+            requireAuthentication=False,
         )
-        # TODO: Change this to connectSSL when cert auth is in place in the platform
+
         reactor.connectTCP(
             self._config[HOST_KEY],
             self._config[PORT_KEY],