Merge branch 'release-0.3.3'0.3.3

3 files changed, 172 insertions, 123 deletions

diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py
index 0520c7c..b80ccfd 100644
--- a/src/leap/mx/check_recipient_access.py
+++ b/src/leap/mx/check_recipient_access.py
@@ -29,11 +29,13 @@ from leap.mx.alias_resolver import AliasResolverFactory
 
 class LEAPPostFixTCPMapserverAccess(postfix.PostfixTCPMapServer):
     def _cbGot(self, value):
+        # For more info, see:
+        # http://www.postfix.org/tcp_table.5.html
+        # http://www.postfix.org/access.5.html
         if value is None:
-            self.sendCode(500, postfix.quote("NOT FOUND SORRY"))
+            self.sendCode(500, postfix.quote("REJECT"))
         else:
-            # We do not send the value in this case
-            self.sendCode(200)
+            self.sendCode(200, postfix.quote("OK"))
 
 
 class CheckRecipientAccessFactory(AliasResolverFactory):
diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py
index 147e6f9..41604ba 100644
--- a/src/leap/mx/couchdbhelper.py
+++ b/src/leap/mx/couchdbhelper.py
@@ -29,7 +29,6 @@ except ImportError:
     print "for instructions on getting required dependencies."
 
 try:
-    from twisted.internet import defer
     from twisted.python import log
 except ImportError:
     print "This software requires Twisted. Please see the README file"
@@ -49,16 +48,16 @@ class ConnectedCouchDB(client.CouchDB):
         """
         Connect to a CouchDB instance.
 
-        @param host: A hostname string for the CouchDB server.
-        @type host: str
-        @param port: The port of the CouchDB server.
-        @type port: int
-        @param dbName: (optional) The default database to bind queries to.
-        @type dbName: str
-        @param username: (optional) The username for authorization.
-        @type username: str
-        @param str password: (optional) The password for authorization.
-        @type password: str
+        :param host: A hostname string for the CouchDB server.
+        :type host: str
+        :param port: The port of the CouchDB server.
+        :type port: int
+        :param dbName: (optional) The default database to bind queries to.
+        :type dbName: str
+        :param username: (optional) The username for authorization.
+        :type username: str
+        :param str password: (optional) The password for authorization.
+        :type password: str
         """
         client.CouchDB.__init__(self,
                                 host,
@@ -78,8 +77,8 @@ class ConnectedCouchDB(client.CouchDB):
         """
         Callback for listDB that prints the available databases
 
-        @param data: response from the listDB command
-        @type data: array
+        :param data: response from the listDB command
+        :type data: array
         """
         log.msg("Available databases:")
         for database in data:
@@ -101,10 +100,10 @@ class ConnectedCouchDB(client.CouchDB):
         """
         Check to see if a particular email or alias exists.
 
-        @param alias: A string representing the email or alias to check.
-        @type alias: str
-        @return: a deferred for this query
-        @rtype twisted.defer.Deferred
+        :param alias: A string representing the email or alias to check.
+        :type alias: str
+        :return: a deferred for this query
+        :rtype twisted.defer.Deferred
         """
         assert isinstance(address, (str, unicode)), "Email or alias queries must be string"
 
@@ -124,12 +123,12 @@ class ConnectedCouchDB(client.CouchDB):
         """
         Parses the result of the by_address query and gets the uuid
 
-        @param address: alias looked up
-        @type address: string
-        @param result: result dictionary
-        @type result: dict
-        @return: The uuid for alias if available
-        @rtype: str
+        :param address: alias looked up
+        :type address: string
+        :param result: result dictionary
+        :type result: dict
+        :return: The uuid for alias if available
+        :rtype: str
         """
         for row in result["rows"]:
             if row["key"] == address:
@@ -140,20 +139,39 @@ class ConnectedCouchDB(client.CouchDB):
                 return uuid
         return None
 
-    def getPubKey(self, address):
+    def getPubKey(self, uuid):
+        """
+        Returns a deferred that will return the pubkey for the uuid provided
+
+        :param uuid: uuid for the user to query
+        :type uuid: str
+
+        :rtype: Deferred
+        """
         d = self.openView(docId="Identity",
                           viewId="pgp_key_by_email/",
-                          key=address,
+                          user_id=uuid,
                           reduce=False,
                           include_docs=True)
 
-        d.addCallbacks(partial(self._get_pgp_key, address), log.err)
+        d.addCallbacks(partial(self._get_pgp_key, uuid), log.err)
 
         return d
 
-    def _get_pgp_key(self, address, result):
+    def _get_pgp_key(self, uuid, result):
+        """
+        Callback used to filter the correct pubkey from the result of
+        the query to the couchdb
+
+        :param uuid: uuid for the user that was queried
+        :type uuid: str
+        :param result: result dictionary for the db query
+        :type result: dict
+
+        :rtype: str or None
+        """
         for row in result["rows"]:
-            if row["key"] == address:
+            if row["doc"]["user_id"] == uuid:
                 return row["value"]
         return None
 
diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py
index 5875034..8fcadce 100644
--- a/src/leap/mx/mail_receiver.py
+++ b/src/leap/mx/mail_receiver.py
@@ -25,14 +25,20 @@ import uuid as pyuuid
 
 import json
 import email.utils
+import socket
+
+try:
+    import cchardet as chardet
+except ImportError:
+    import chardet
 
 from email import message_from_string
 
 from twisted.application.service import Service
 from twisted.internet import inotify
-from twisted.internet.defer import DeferredList
 from twisted.python import filepath, log
 
+from leap.common.mail import get_email_charset
 from leap.soledad.common.document import SoledadDocument
 from leap.soledad.common.crypto import (
     EncryptionSchemes,
@@ -54,19 +60,20 @@ class MailReceiver(Service):
         """
         Constructor
 
-        @param mail_couch_url: URL prefix for the couchdb where mail
+        :param mail_couch_url: URL prefix for the couchdb where mail
         should be stored
-        @type mail_couch_url: str
-        @param users_cdb: CouchDB instance from where to get the uuid
+        :type mail_couch_url: str
+        :param users_cdb: CouchDB instance from where to get the uuid
         and pubkey for a user
-        @type users_cdb: ConnectedCouchDB
-        @param directories: list of directories to monitor
-        @type directories: list of tuples (path: str, recursive: bool)
+        :type users_cdb: ConnectedCouchDB
+        :param directories: list of directories to monitor
+        :type directories: list of tuples (path: str, recursive: bool)
         """
         # Service doesn't define an __init__
         self._mail_couch_url = mail_couch_url
         self._users_cdb = users_cdb
         self._directories = directories
+        self._domain = socket.gethostbyaddr(socket.gethostname())[0]
 
     def startService(self):
         """
@@ -79,56 +86,49 @@ class MailReceiver(Service):
         mask = inotify.IN_CREATE
 
         for directory, recursive in self._directories:
-            log.msg("Watching %s --- Recursive: %s" % (directory, recursive))
+            log.msg("Watching %r --- Recursive: %r" % (directory, recursive))
             self.wm.watch(filepath.FilePath(directory), mask,
                           callbacks=[self._process_incoming_email],
                           recursive=recursive)
 
-    def _gather_uuid_pubkey(self, results):
-        if len(results) < 2:
-            return None, None
-
-        # DeferredList results are structured like this:
-        # [ (succeeded, pubkey), (succeeded, uuid) ]
-        # succeeded is a bool value that specifies if the
-        # corresponding callback succeeded
-        pubkey_res, uuid_res = results
-
-        pubkey = pubkey_res[1] if pubkey_res[0] else None
-        uuid = uuid_res[1] if uuid_res[0] else None
-
-        return uuid, pubkey
-
-    def _encrypt_message(self, uuid_pubkey, address, message):
+    def _encrypt_message(self, pubkey, uuid, message):
         """
-        Given a UUID, a public key, address and a message, it encrypts
-        the message to that public key.
+        Given a UUID, a public key and a message, it encrypts the
+        message to that public key.
         The address is needed in order to build the OpenPGPKey object.
 
-        @param uuid_pubkey: tuple that holds the uuid and the public
-        key as it is returned by the previous call in the chain
-        @type uuid_pubkey: tuple (str, str)
-        @param address: mail address for this message
-        @type address: str
-        @param message: message contents
-        @type message: str
+        :param uuid_pubkey: tuple that holds the uuid and the public
+                            key as it is returned by the previous call in the
+                            chain
+        :type uuid_pubkey: tuple (str, str)
+        :param message: message contents
+        :type message: str
 
-        @return: uuid, doc to sync with Soledad
-        @rtype: tuple(str, SoledadDocument)
+        :return: uuid, doc to sync with Soledad or None, None if
+                 something went wrong.
+        :rtype: tuple(str, SoledadDocument)
         """
-        uuid, pubkey = uuid_pubkey
+        if uuid is None or pubkey is None or len(pubkey) == 0:
+            log.msg("_encrypt_message: Something went wrong, here's all "
+                    "I know: %r | %r" % (uuid, pubkey))
+            return None, None
+
         log.msg("Encrypting message to %s's pubkey" % (uuid,))
-        log.msg("Pubkey: %s" % (pubkey,))
 
         doc = SoledadDocument(doc_id=str(pyuuid.uuid4()))
 
+        encoding = get_email_charset(message, default=None)
+        if encoding is None:
+            result = chardet.detect(message)
+            encoding = result["encoding"]
+
         data = {'incoming': True, 'content': message}
 
         if pubkey is None or len(pubkey) == 0:
             doc.content = {
                 self.INCOMING_KEY: True,
                 ENC_SCHEME_KEY: EncryptionSchemes.NONE,
-                ENC_JSON_KEY: json.dumps(data)
+                ENC_JSON_KEY: json.dumps(data, encoding=encoding)
             }
             return uuid, doc
 
@@ -136,13 +136,15 @@ class MailReceiver(Service):
         with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg:
             gpg.import_keys(pubkey)
             key = gpg.list_keys().pop()
-            openpgp_key = openpgp._build_key_from_gpg(address, key, pubkey)
+            # We don't care about the actual address, so we use a
+            # dummy one, we just care about the import of the pubkey
+            openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com", key, pubkey)
 
             doc.content = {
                 self.INCOMING_KEY: True,
                 ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
                 ENC_JSON_KEY: str(gpg.encrypt(
-                    json.dumps(data),
+                    json.dumps(data, encoding=encoding),
                     openpgp_key.fingerprint,
                     symmetric=False))
             }
@@ -153,20 +155,22 @@ class MailReceiver(Service):
         """
         Given a UUID and a SoledadDocument, it saves it directly in the
         couchdb that serves as a backend for Soledad, in a db
-        accessible to the recipient of the mail
+        accessible to the recipient of the mail.
 
-        @param uuid_doc: tuple that holds the UUID and SoledadDocument
-        @type uuid_doc: tuple(str, SoledadDocument)
+        :param uuid_doc: tuple that holds the UUID and SoledadDocument
+        :type uuid_doc: tuple(str, SoledadDocument)
 
-        @return: True if it's ok to remove the message, False
-        otherwise
-        @rtype: bool
+        :return: True if it's ok to remove the message, False
+                 otherwise
+        :rtype: bool
         """
         uuid, doc = uuid_doc
-        log.msg("Exporting message for %s" % (uuid,))
+        if uuid is None or doc is None:
+            log.msg("_export_message: Something went wrong, here's all "
+                    "I know: %r | %r" % (uuid, doc))
+            return False
 
-        if uuid is None:
-            uuid = 0
+        log.msg("Exporting message for %s" % (uuid,))
 
         db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,))
         db.put_doc(doc)
@@ -177,58 +181,83 @@ class MailReceiver(Service):
 
     def _conditional_remove(self, do_remove, filepath):
         """
-        Removes the message if do_remove is True
+        Removes the message if do_remove is True.
 
-        @param do_remove: True if the message should be removed, False
-        otherwise
-        @type do_remove: bool
-        @param filepath: path to the mail
-        @type filepath: twisted.python.filepath.FilePath
+        :param do_remove: True if the message should be removed, False
+                          otherwise
+        :type do_remove: bool
+        :param filepath: path to the mail
+        :type filepath: twisted.python.filepath.FilePath
         """
         if do_remove:
             # remove the original mail
             try:
-                log.msg("Removing %s" % (filepath.path,))
+                log.msg("Removing %r" % (filepath.path,))
                 filepath.remove()
                 log.msg("Done removing")
-            except:
+            except Exception:
                 log.err()
+        else:
+            log.msg("Not removing %r" % (filepath.path,))
+
+    def _get_owner(self, mail):
+        """
+        Given an email, returns the uuid of the owner.
+
+        :param mail: mail to analyze
+        :type mail: email.message.Message
+
+        :returns: uuid
+        :rtype: str or None
+        """
+        uuid = None
+
+        delivereds = mail.get_all("Delivered-To")
+        for to in delivereds:
+            name, addr = email.utils.parseaddr(to)
+            parts = addr.split("@")
+            if len(parts) > 1 and parts[1] == self._domain:
+                uuid = parts[0]
+                break
+
+        return uuid
 
     def _process_incoming_email(self, otherself, filepath, mask):
         """
-        Callback that processes incoming email
-
-        @param otherself: Watch object for the current callback from
-        inotify
-        @type otherself: twisted.internet.inotify._Watch
-        @param filepath: Path of the file that changed
-        @type filepath: twisted.python.filepath.FilePath
-        @param mask: identifier for the type of change that triggered
-        this callback
-        @type mask: int
+        Callback that processes incoming email.
+
+        :param otherself: Watch object for the current callback from
+                          inotify.
+        :type otherself: twisted.internet.inotify._Watch
+        :param filepath: Path of the file that changed
+        :type filepath: twisted.python.filepath.FilePath
+        :param mask: identifier for the type of change that triggered
+                     this callback
+        :type mask: int
         """
-        if os.path.split(filepath.dirname())[-1]  == "new":
-            log.msg("Processing new mail at %s" % (filepath.path,))
-            with filepath.open("r") as f:
-                mail_data = f.read()
-                mail = message_from_string(mail_data)
-                owner = mail["To"]
-                if owner is None:  # default to Delivered-To
-                    owner = mail["Delivered-To"]
-                if owner is None:
-                    log.err("Malformed mail, neither To: nor "
-                            "Delivered-To: field")
-                log.msg("Mail owner: %s" % (owner,))
-
-                owner = email.utils.parseaddr(owner)[1]
-                log.msg("%s received a new mail" % (owner,))
-                dpubk = self._users_cdb.getPubKey(owner)
-                duuid = self._users_cdb.queryByAddress(owner)
-                d = DeferredList([dpubk, duuid])
-                d.addCallbacks(self._gather_uuid_pubkey, log.err)
-                d.addCallbacks(self._encrypt_message, log.err,
-                               (owner, mail_data))
-                d.addCallbacks(self._export_message, log.err)
-                d.addCallbacks(self._conditional_remove, log.err,
-                               (filepath,))
-                d.addErrback(log.err)
+        try:
+            if os.path.split(filepath.dirname())[-1]  == "new":
+                log.msg("Processing new mail at %r" % (filepath.path,))
+                with filepath.open("r") as f:
+                    mail_data = f.read()
+                    mail = message_from_string(mail_data)
+                    uuid = self._get_owner(mail)
+                    if uuid is None:
+                        log.msg("Don't know how to deliver mail %r, skipping..." %
+                                filepath.path)
+                        return
+                    log.msg("Mail owner: %s" % (uuid,))
+
+                    if uuid is None:
+                        log.msg("BUG: There was no uuid!")
+                        return
+
+                    d = self._users_cdb.getPubKey(uuid)
+                    d.addCallbacks(self._encrypt_message, log.err,
+                                   (uuid, mail_data))
+                    d.addCallbacks(self._export_message, log.err)
+                    d.addCallbacks(self._conditional_remove, log.err,
+                                   (filepath,))
+                    d.addErrback(log.err)
+        except Exception:
+            log.err()