Merge remote-tracking branch 'chiiph/bug/support_mailing_lists_and_encodings' into develop

4 files changed, 115 insertions, 53 deletions

diff --git a/changes/bug_support_more_than_utf8 b/changes/bug_support_more_than_utf8
new file mode 100644
index 0000000..2e467aa
--- /dev/null
+++ b/changes/bug_support_more_than_utf8
@@ -0,0 +1,5 @@
+  o Support more than utf8 encodings for emails.
+  o Add support for receiving mailing list mails.
+  o Use the uuid that alias_resolver returned and postfix added to the
+    mail headers, which improves performance.
+  o Look for public keys based on uuid instead of mail address.
\ No newline at end of file
diff --git a/pkg/requirements.pip b/pkg/requirements.pip
index d5db275..06e3b85 100644
--- a/pkg/requirements.pip
+++ b/pkg/requirements.pip
@@ -12,3 +12,5 @@ couchdb
 python-gnupg>=0.3.0
 leap.soledad.common>=0.3.0
 leap.keymanager>=0.2.0
+
+cchardet  # we fallback to chardet if this is not available, but it's preferred
\ No newline at end of file
diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py
index b500f17..41604ba 100644
--- a/src/leap/mx/couchdbhelper.py
+++ b/src/leap/mx/couchdbhelper.py
@@ -29,7 +29,6 @@ except ImportError:
     print "for instructions on getting required dependencies."
 
 try:
-    from twisted.internet import defer
     from twisted.python import log
 except ImportError:
     print "This software requires Twisted. Please see the README file"
@@ -140,20 +139,39 @@ class ConnectedCouchDB(client.CouchDB):
                 return uuid
         return None
 
-    def getPubKey(self, address):
+    def getPubKey(self, uuid):
+        """
+        Returns a deferred that will return the pubkey for the uuid provided
+
+        :param uuid: uuid for the user to query
+        :type uuid: str
+
+        :rtype: Deferred
+        """
         d = self.openView(docId="Identity",
                           viewId="pgp_key_by_email/",
-                          key=address,
+                          user_id=uuid,
                           reduce=False,
                           include_docs=True)
 
-        d.addCallbacks(partial(self._get_pgp_key, address), log.err)
+        d.addCallbacks(partial(self._get_pgp_key, uuid), log.err)
 
         return d
 
-    def _get_pgp_key(self, address, result):
+    def _get_pgp_key(self, uuid, result):
+        """
+        Callback used to filter the correct pubkey from the result of
+        the query to the couchdb
+
+        :param uuid: uuid for the user that was queried
+        :type uuid: str
+        :param result: result dictionary for the db query
+        :type result: dict
+
+        :rtype: str or None
+        """
         for row in result["rows"]:
-            if row["key"] == address:
+            if row["doc"]["user_id"] == uuid:
                 return row["value"]
         return None
 
diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py
index e263604..bae489f 100644
--- a/src/leap/mx/mail_receiver.py
+++ b/src/leap/mx/mail_receiver.py
@@ -15,20 +15,27 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 """
 MailReceiver service definition
 """
+
 import os
 import uuid as pyuuid
 
 import json
 import email.utils
+import socket
+
+try:
+    import cchardet as chardet
+except ImportError:
+    import chardet
 
 from email import message_from_string
 
 from twisted.application.service import Service
 from twisted.internet import inotify
-from twisted.internet.defer import DeferredList
 from twisted.python import filepath, log
 
 from leap.soledad.common.document import SoledadDocument
@@ -65,6 +72,7 @@ class MailReceiver(Service):
         self._mail_couch_url = mail_couch_url
         self._users_cdb = users_cdb
         self._directories = directories
+        self._domain = socket.gethostbyaddr(socket.gethostname())[0]
 
     def startService(self):
         """
@@ -82,22 +90,7 @@ class MailReceiver(Service):
                           callbacks=[self._process_incoming_email],
                           recursive=recursive)
 
-    def _gather_uuid_pubkey(self, results):
-        if len(results) < 2:
-            return None, None
-
-        # DeferredList results are structured like this:
-        # [ (succeeded, pubkey), (succeeded, uuid) ]
-        # succeeded is a bool value that specifies if the
-        # corresponding callback succeeded
-        pubkey_res, uuid_res = results
-
-        pubkey = pubkey_res[1] if pubkey_res[0] else None
-        uuid = uuid_res[1] if uuid_res[0] else None
-
-        return uuid, pubkey
-
-    def _encrypt_message(self, uuid_pubkey, address, message):
+    def _encrypt_message(self, pubkey, uuid, address, message):
         """
         Given a UUID, a public key, address and a message, it encrypts
         the message to that public key.
@@ -112,22 +105,29 @@ class MailReceiver(Service):
         :param message: message contents
         :type message: str
 
-        :return: uuid, doc to sync with Soledad
+        :return: uuid, doc to sync with Soledad or None, None if
+                 something went wrong.
         :rtype: tuple(str, SoledadDocument)
         """
-        uuid, pubkey = uuid_pubkey
+        if uuid is None or pubkey is None or len(pubkey) == 0:
+            return None, None
+
         log.msg("Encrypting message to %s's pubkey" % (uuid,))
         log.msg("Pubkey: %s" % (pubkey,))
 
         doc = SoledadDocument(doc_id=str(pyuuid.uuid4()))
 
+        result = chardet.detect(message)
+
+        encoding = result["encoding"]
+
         data = {'incoming': True, 'content': message}
 
         if pubkey is None or len(pubkey) == 0:
             doc.content = {
                 self.INCOMING_KEY: True,
                 ENC_SCHEME_KEY: EncryptionSchemes.NONE,
-                ENC_JSON_KEY: json.dumps(data)
+                ENC_JSON_KEY: json.dumps(data, encoding=encoding)
             }
             return uuid, doc
 
@@ -141,7 +141,7 @@ class MailReceiver(Service):
                 self.INCOMING_KEY: True,
                 ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
                 ENC_JSON_KEY: str(gpg.encrypt(
-                    json.dumps(data),
+                    json.dumps(data, encoding=encoding),
                     openpgp_key.fingerprint,
                     symmetric=False))
             }
@@ -162,6 +162,9 @@ class MailReceiver(Service):
         :rtype: bool
         """
         uuid, doc = uuid_doc
+        if uuid is None or doc is None:
+            return False
+
         log.msg("Exporting message for %s" % (uuid,))
 
         if uuid is None:
@@ -190,9 +193,41 @@ class MailReceiver(Service):
                 log.msg("Removing %s" % (filepath.path,))
                 filepath.remove()
                 log.msg("Done removing")
-            except:
+            except Exception:
                 log.err()
 
+    def _get_owner(self, mail):
+        """
+        Given an email, returns the owner (who's delivered to) and the
+        uuid of the owner. In case of a mailing list mail, the owner
+        will end up being the mailing list address, but the owner's
+        uuid will be correct as long as the user exists in the system.
+
+        :param mail: mail to analyze
+        :type mail: email.message.Message
+
+        :returns: a tuple of (owner, uuid)
+        :rtype: tuple(str or None, str or None)
+        """
+        owner = mail["To"]
+        uuid = None
+
+        delivereds = mail.get_all("Delivered-To")
+        for to in delivereds:
+            name, addr = email.utils.parseaddr(to)
+            parts = addr.split("@")
+            if len(parts) > 1 and parts[1] == self._domain:
+                uuid = parts[0]
+                break
+
+        if owner is None:  # default to Delivered-To
+            owner = mail["Delivered-To"]
+        if owner is None:
+            log.err("Malformed mail, neither To: nor "
+                    "Delivered-To: field")
+
+        return owner, uuid
+
     def _process_incoming_email(self, otherself, filepath, mask):
         """
         Callback that processes incoming email.
@@ -206,28 +241,30 @@ class MailReceiver(Service):
                      this callback
         :type mask: int
         """
-        if os.path.split(filepath.dirname())[-1] == "new":
-            log.msg("Processing new mail at %s" % (filepath.path,))
-            with filepath.open("r") as f:
-                mail_data = f.read()
-                mail = message_from_string(mail_data)
-                owner = mail["To"]
-                if owner is None:  # default to Delivered-To
-                    owner = mail["Delivered-To"]
-                if owner is None:
-                    log.err("Malformed mail, neither To: nor "
-                            "Delivered-To: field")
-                log.msg("Mail owner: %s" % (owner,))
-
-                owner = email.utils.parseaddr(owner)[1]
-                log.msg("%s received a new mail" % (owner,))
-                dpubk = self._users_cdb.getPubKey(owner)
-                duuid = self._users_cdb.queryByAddress(owner)
-                d = DeferredList([dpubk, duuid])
-                d.addCallbacks(self._gather_uuid_pubkey, log.err)
-                d.addCallbacks(self._encrypt_message, log.err,
-                               (owner, mail_data))
-                d.addCallbacks(self._export_message, log.err)
-                d.addCallbacks(self._conditional_remove, log.err,
-                               (filepath,))
-                d.addErrback(log.err)
+        try:
+            if os.path.split(filepath.dirname())[-1]  == "new":
+                log.msg("Processing new mail at %s" % (filepath.path,))
+                with filepath.open("r") as f:
+                    mail_data = f.read()
+                    mail = message_from_string(mail_data)
+                    owner, uuid = self._get_owner(mail)
+                    if owner is None:
+                        # This shouldn't happen, may be a bug in
+                        # postfix? But we don't want to drop mail just
+                        # because of a bug. Skipping this one...
+                        return
+                    log.msg("Mail owner: %s %s" % (owner, uuid))
+
+                    if uuid is None:
+                        log.msg("BUG: There was no uuid!")
+                        return
+
+                    d = self._users_cdb.getPubKey(uuid)
+                    d.addCallbacks(self._encrypt_message, log.err,
+                                   (uuid, owner, mail_data))
+                    d.addCallbacks(self._export_message, log.err)
+                    d.addCallbacks(self._conditional_remove, log.err,
+                                   (filepath,))
+                    d.addErrback(log.err)
+        except Exception:
+            log.err()