Merge branch 'release-0.3.3'0.3.3

7 files changed, 266 insertions, 128 deletions

diff --git a/CHANGELOG b/CHANGELOG
index fcc216e..67ff1e5 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,15 @@
+0.3.3 Nov 1:
+  o Fix return codes for check recipient access. Fixes #3356.
+  o Improve logging in general and support possible unicode parameters
+    without breaking.
+  o Try to figure out the encoding of an email first by looking into
+    its header, if that fails then by using chardet.
+  o Support more than utf8 encodings for emails.
+  o Add support for receiving mailing list mails.
+  o Use the uuid that alias_resolver returned and postfix added to the
+    mail headers, which improves performance.
+  o Look for public keys based on uuid instead of mail address.
+
 0.3.2 Sep 6:
   o Keep file watcher in memory to prevent losing file events.
   o Properly save the incoming mail as a doc in couch.
diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT
new file mode 100644
index 0000000..cc00ecf
--- /dev/null
+++ b/changes/VERSION_COMPAT
@@ -0,0 +1,10 @@
+#################################################
+# This file keeps track of the recent changes
+# introduced in internal leap dependencies.
+# Add your changes here so we can properly update
+# requirements.pip during the release process.
+# (leave header when resetting)
+#################################################
+#
+# BEGIN DEPENDENCY LIST -------------------------
+# leap.foo.bar>=x.y.z
diff --git a/pkg/requirements.pip b/pkg/requirements.pip
index d5db275..4242ad4 100644
--- a/pkg/requirements.pip
+++ b/pkg/requirements.pip
@@ -8,7 +8,8 @@ paisley>=0.3.1
 # for the time being.
 couchdb
 
-## XXX change me to whatever you name the package in pypi
-python-gnupg>=0.3.0
+leap.common>=0.3.5
 leap.soledad.common>=0.3.0
-leap.keymanager>=0.2.0
+leap.keymanager>=0.3.4
+
+cchardet  # we fallback to chardet if this is not available, but it's preferred
diff --git a/setup.py b/setup.py
index 3de66cf..6fec416 100644
--- a/setup.py
+++ b/setup.py
@@ -18,6 +18,7 @@
 setup file for leap.mx
 """
 import os
+import re
 from setuptools import setup, find_packages
 
 import versioneer
@@ -43,6 +44,68 @@ trove_classifiers = [
     'Topic :: Security :: Cryptography',
 ]
 
+DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mx/'
+                 'archive/%s.tar.gz')
+_versions = versioneer.get_versions()
+VERSION = _versions['version']
+VERSION_FULL = _versions['full']
+DOWNLOAD_URL = ""
+
+# get the short version for the download url
+_version_short = re.findall('\d+\.\d+\.\d+', VERSION)
+if len(_version_short) > 0:
+    VERSION_SHORT = _version_short[0]
+    DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT
+
+cmdclass = versioneer.get_cmdclass()
+
+
+from setuptools import Command
+
+
+class freeze_debianver(Command):
+    """
+    Freezes the version in a debian branch.
+    To be used after merging the development branch onto the debian one.
+    """
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        proceed = str(raw_input(
+            "This will overwrite the file _version.py. Continue? [y/N] "))
+        if proceed != "y":
+            print("He. You scared. Aborting.")
+            return
+        template = r"""
+# This file was generated by the `freeze_debianver` command in setup.py
+# Using 'versioneer.py' (0.7+) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+version_version = '{version}'
+version_full = '{version_full}'
+"""
+        templatefun = r"""
+
+def get_versions(default={}, verbose=False):
+        return {'version': version_version, 'full': version_full}
+"""
+        subst_template = template.format(
+            version=VERSION_SHORT,
+            version_full=VERSION_FULL) + templatefun
+        with open(versioneer.versionfile_source, 'w') as f:
+            f.write(subst_template)
+
+
+cmdclass["freeze_debianver"] = freeze_debianver
+
 if os.environ.get("VIRTUAL_ENV", None):
     data_files = None
 else:
@@ -54,12 +117,15 @@ else:
                   ("/etc/init.d/", ["pkg/leap_mx"])]
 setup(
     name='leap.mx',
-    version=versioneer.get_version(),
-    cmdclass=versioneer.get_cmdclass(),
+    version=VERSION,
+    cmdclass=cmdclass,
     url="http://github.com/leapcode/leap_mx",
+    download_url=DOWNLOAD_URL,
     license='AGPLv3+',
     author='The LEAP Encryption Access Project',
     author_email='info@leap.se',
+    maintainer='Kali Kaneko',
+    maintainer_email='kali@leap.se',
     description=("An asynchronous, transparently-encrypting remailer "
                  "for the LEAP platform"),
     long_description=(
diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py
index 0520c7c..b80ccfd 100644
--- a/src/leap/mx/check_recipient_access.py
+++ b/src/leap/mx/check_recipient_access.py
@@ -29,11 +29,13 @@ from leap.mx.alias_resolver import AliasResolverFactory
 
 class LEAPPostFixTCPMapserverAccess(postfix.PostfixTCPMapServer):
     def _cbGot(self, value):
+        # For more info, see:
+        # http://www.postfix.org/tcp_table.5.html
+        # http://www.postfix.org/access.5.html
         if value is None:
-            self.sendCode(500, postfix.quote("NOT FOUND SORRY"))
+            self.sendCode(500, postfix.quote("REJECT"))
         else:
-            # We do not send the value in this case
-            self.sendCode(200)
+            self.sendCode(200, postfix.quote("OK"))
 
 
 class CheckRecipientAccessFactory(AliasResolverFactory):
diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py
index 147e6f9..41604ba 100644
--- a/src/leap/mx/couchdbhelper.py
+++ b/src/leap/mx/couchdbhelper.py
@@ -29,7 +29,6 @@ except ImportError:
     print "for instructions on getting required dependencies."
 
 try:
-    from twisted.internet import defer
     from twisted.python import log
 except ImportError:
     print "This software requires Twisted. Please see the README file"
@@ -49,16 +48,16 @@ class ConnectedCouchDB(client.CouchDB):
         """
         Connect to a CouchDB instance.
 
-        @param host: A hostname string for the CouchDB server.
-        @type host: str
-        @param port: The port of the CouchDB server.
-        @type port: int
-        @param dbName: (optional) The default database to bind queries to.
-        @type dbName: str
-        @param username: (optional) The username for authorization.
-        @type username: str
-        @param str password: (optional) The password for authorization.
-        @type password: str
+        :param host: A hostname string for the CouchDB server.
+        :type host: str
+        :param port: The port of the CouchDB server.
+        :type port: int
+        :param dbName: (optional) The default database to bind queries to.
+        :type dbName: str
+        :param username: (optional) The username for authorization.
+        :type username: str
+        :param str password: (optional) The password for authorization.
+        :type password: str
         """
         client.CouchDB.__init__(self,
                                 host,
@@ -78,8 +77,8 @@ class ConnectedCouchDB(client.CouchDB):
         """
         Callback for listDB that prints the available databases
 
-        @param data: response from the listDB command
-        @type data: array
+        :param data: response from the listDB command
+        :type data: array
         """
         log.msg("Available databases:")
         for database in data:
@@ -101,10 +100,10 @@ class ConnectedCouchDB(client.CouchDB):
         """
         Check to see if a particular email or alias exists.
 
-        @param alias: A string representing the email or alias to check.
-        @type alias: str
-        @return: a deferred for this query
-        @rtype twisted.defer.Deferred
+        :param alias: A string representing the email or alias to check.
+        :type alias: str
+        :return: a deferred for this query
+        :rtype twisted.defer.Deferred
         """
         assert isinstance(address, (str, unicode)), "Email or alias queries must be string"
 
@@ -124,12 +123,12 @@ class ConnectedCouchDB(client.CouchDB):
         """
         Parses the result of the by_address query and gets the uuid
 
-        @param address: alias looked up
-        @type address: string
-        @param result: result dictionary
-        @type result: dict
-        @return: The uuid for alias if available
-        @rtype: str
+        :param address: alias looked up
+        :type address: string
+        :param result: result dictionary
+        :type result: dict
+        :return: The uuid for alias if available
+        :rtype: str
         """
         for row in result["rows"]:
             if row["key"] == address:
@@ -140,20 +139,39 @@ class ConnectedCouchDB(client.CouchDB):
                 return uuid
         return None
 
-    def getPubKey(self, address):
+    def getPubKey(self, uuid):
+        """
+        Returns a deferred that will return the pubkey for the uuid provided
+
+        :param uuid: uuid for the user to query
+        :type uuid: str
+
+        :rtype: Deferred
+        """
         d = self.openView(docId="Identity",
                           viewId="pgp_key_by_email/",
-                          key=address,
+                          user_id=uuid,
                           reduce=False,
                           include_docs=True)
 
-        d.addCallbacks(partial(self._get_pgp_key, address), log.err)
+        d.addCallbacks(partial(self._get_pgp_key, uuid), log.err)
 
         return d
 
-    def _get_pgp_key(self, address, result):
+    def _get_pgp_key(self, uuid, result):
+        """
+        Callback used to filter the correct pubkey from the result of
+        the query to the couchdb
+
+        :param uuid: uuid for the user that was queried
+        :type uuid: str
+        :param result: result dictionary for the db query
+        :type result: dict
+
+        :rtype: str or None
+        """
         for row in result["rows"]:
-            if row["key"] == address:
+            if row["doc"]["user_id"] == uuid:
                 return row["value"]
         return None
 
diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py
index 5875034..8fcadce 100644
--- a/src/leap/mx/mail_receiver.py
+++ b/src/leap/mx/mail_receiver.py
@@ -25,14 +25,20 @@ import uuid as pyuuid
 
 import json
 import email.utils
+import socket
+
+try:
+    import cchardet as chardet
+except ImportError:
+    import chardet
 
 from email import message_from_string
 
 from twisted.application.service import Service
 from twisted.internet import inotify
-from twisted.internet.defer import DeferredList
 from twisted.python import filepath, log
 
+from leap.common.mail import get_email_charset
 from leap.soledad.common.document import SoledadDocument
 from leap.soledad.common.crypto import (
     EncryptionSchemes,
@@ -54,19 +60,20 @@ class MailReceiver(Service):
         """
         Constructor
 
-        @param mail_couch_url: URL prefix for the couchdb where mail
+        :param mail_couch_url: URL prefix for the couchdb where mail
         should be stored
-        @type mail_couch_url: str
-        @param users_cdb: CouchDB instance from where to get the uuid
+        :type mail_couch_url: str
+        :param users_cdb: CouchDB instance from where to get the uuid
         and pubkey for a user
-        @type users_cdb: ConnectedCouchDB
-        @param directories: list of directories to monitor
-        @type directories: list of tuples (path: str, recursive: bool)
+        :type users_cdb: ConnectedCouchDB
+        :param directories: list of directories to monitor
+        :type directories: list of tuples (path: str, recursive: bool)
         """
         # Service doesn't define an __init__
         self._mail_couch_url = mail_couch_url
         self._users_cdb = users_cdb
         self._directories = directories
+        self._domain = socket.gethostbyaddr(socket.gethostname())[0]
 
     def startService(self):
         """
@@ -79,56 +86,49 @@ class MailReceiver(Service):
         mask = inotify.IN_CREATE
 
         for directory, recursive in self._directories:
-            log.msg("Watching %s --- Recursive: %s" % (directory, recursive))
+            log.msg("Watching %r --- Recursive: %r" % (directory, recursive))
             self.wm.watch(filepath.FilePath(directory), mask,
                           callbacks=[self._process_incoming_email],
                           recursive=recursive)
 
-    def _gather_uuid_pubkey(self, results):
-        if len(results) < 2:
-            return None, None
-
-        # DeferredList results are structured like this:
-        # [ (succeeded, pubkey), (succeeded, uuid) ]
-        # succeeded is a bool value that specifies if the
-        # corresponding callback succeeded
-        pubkey_res, uuid_res = results
-
-        pubkey = pubkey_res[1] if pubkey_res[0] else None
-        uuid = uuid_res[1] if uuid_res[0] else None
-
-        return uuid, pubkey
-
-    def _encrypt_message(self, uuid_pubkey, address, message):
+    def _encrypt_message(self, pubkey, uuid, message):
         """
-        Given a UUID, a public key, address and a message, it encrypts
-        the message to that public key.
+        Given a UUID, a public key and a message, it encrypts the
+        message to that public key.
         The address is needed in order to build the OpenPGPKey object.
 
-        @param uuid_pubkey: tuple that holds the uuid and the public
-        key as it is returned by the previous call in the chain
-        @type uuid_pubkey: tuple (str, str)
-        @param address: mail address for this message
-        @type address: str
-        @param message: message contents
-        @type message: str
+        :param uuid_pubkey: tuple that holds the uuid and the public
+                            key as it is returned by the previous call in the
+                            chain
+        :type uuid_pubkey: tuple (str, str)
+        :param message: message contents
+        :type message: str
 
-        @return: uuid, doc to sync with Soledad
-        @rtype: tuple(str, SoledadDocument)
+        :return: uuid, doc to sync with Soledad or None, None if
+                 something went wrong.
+        :rtype: tuple(str, SoledadDocument)
         """
-        uuid, pubkey = uuid_pubkey
+        if uuid is None or pubkey is None or len(pubkey) == 0:
+            log.msg("_encrypt_message: Something went wrong, here's all "
+                    "I know: %r | %r" % (uuid, pubkey))
+            return None, None
+
         log.msg("Encrypting message to %s's pubkey" % (uuid,))
-        log.msg("Pubkey: %s" % (pubkey,))
 
         doc = SoledadDocument(doc_id=str(pyuuid.uuid4()))
 
+        encoding = get_email_charset(message, default=None)
+        if encoding is None:
+            result = chardet.detect(message)
+            encoding = result["encoding"]
+
         data = {'incoming': True, 'content': message}
 
         if pubkey is None or len(pubkey) == 0:
             doc.content = {
                 self.INCOMING_KEY: True,
                 ENC_SCHEME_KEY: EncryptionSchemes.NONE,
-                ENC_JSON_KEY: json.dumps(data)
+                ENC_JSON_KEY: json.dumps(data, encoding=encoding)
             }
             return uuid, doc
 
@@ -136,13 +136,15 @@ class MailReceiver(Service):
         with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg:
             gpg.import_keys(pubkey)
             key = gpg.list_keys().pop()
-            openpgp_key = openpgp._build_key_from_gpg(address, key, pubkey)
+            # We don't care about the actual address, so we use a
+            # dummy one, we just care about the import of the pubkey
+            openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com", key, pubkey)
 
             doc.content = {
                 self.INCOMING_KEY: True,
                 ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
                 ENC_JSON_KEY: str(gpg.encrypt(
-                    json.dumps(data),
+                    json.dumps(data, encoding=encoding),
                     openpgp_key.fingerprint,
                     symmetric=False))
             }
@@ -153,20 +155,22 @@ class MailReceiver(Service):
         """
         Given a UUID and a SoledadDocument, it saves it directly in the
         couchdb that serves as a backend for Soledad, in a db
-        accessible to the recipient of the mail
+        accessible to the recipient of the mail.
 
-        @param uuid_doc: tuple that holds the UUID and SoledadDocument
-        @type uuid_doc: tuple(str, SoledadDocument)
+        :param uuid_doc: tuple that holds the UUID and SoledadDocument
+        :type uuid_doc: tuple(str, SoledadDocument)
 
-        @return: True if it's ok to remove the message, False
-        otherwise
-        @rtype: bool
+        :return: True if it's ok to remove the message, False
+                 otherwise
+        :rtype: bool
         """
         uuid, doc = uuid_doc
-        log.msg("Exporting message for %s" % (uuid,))
+        if uuid is None or doc is None:
+            log.msg("_export_message: Something went wrong, here's all "
+                    "I know: %r | %r" % (uuid, doc))
+            return False
 
-        if uuid is None:
-            uuid = 0
+        log.msg("Exporting message for %s" % (uuid,))
 
         db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,))
         db.put_doc(doc)
@@ -177,58 +181,83 @@ class MailReceiver(Service):
 
     def _conditional_remove(self, do_remove, filepath):
         """
-        Removes the message if do_remove is True
+        Removes the message if do_remove is True.
 
-        @param do_remove: True if the message should be removed, False
-        otherwise
-        @type do_remove: bool
-        @param filepath: path to the mail
-        @type filepath: twisted.python.filepath.FilePath
+        :param do_remove: True if the message should be removed, False
+                          otherwise
+        :type do_remove: bool
+        :param filepath: path to the mail
+        :type filepath: twisted.python.filepath.FilePath
         """
         if do_remove:
             # remove the original mail
             try:
-                log.msg("Removing %s" % (filepath.path,))
+                log.msg("Removing %r" % (filepath.path,))
                 filepath.remove()
                 log.msg("Done removing")
-            except:
+            except Exception:
                 log.err()
+        else:
+            log.msg("Not removing %r" % (filepath.path,))
+
+    def _get_owner(self, mail):
+        """
+        Given an email, returns the uuid of the owner.
+
+        :param mail: mail to analyze
+        :type mail: email.message.Message
+
+        :returns: uuid
+        :rtype: str or None
+        """
+        uuid = None
+
+        delivereds = mail.get_all("Delivered-To")
+        for to in delivereds:
+            name, addr = email.utils.parseaddr(to)
+            parts = addr.split("@")
+            if len(parts) > 1 and parts[1] == self._domain:
+                uuid = parts[0]
+                break
+
+        return uuid
 
     def _process_incoming_email(self, otherself, filepath, mask):
         """
-        Callback that processes incoming email
-
-        @param otherself: Watch object for the current callback from
-        inotify
-        @type otherself: twisted.internet.inotify._Watch
-        @param filepath: Path of the file that changed
-        @type filepath: twisted.python.filepath.FilePath
-        @param mask: identifier for the type of change that triggered
-        this callback
-        @type mask: int
+        Callback that processes incoming email.
+
+        :param otherself: Watch object for the current callback from
+                          inotify.
+        :type otherself: twisted.internet.inotify._Watch
+        :param filepath: Path of the file that changed
+        :type filepath: twisted.python.filepath.FilePath
+        :param mask: identifier for the type of change that triggered
+                     this callback
+        :type mask: int
         """
-        if os.path.split(filepath.dirname())[-1]  == "new":
-            log.msg("Processing new mail at %s" % (filepath.path,))
-            with filepath.open("r") as f:
-                mail_data = f.read()
-                mail = message_from_string(mail_data)
-                owner = mail["To"]
-                if owner is None:  # default to Delivered-To
-                    owner = mail["Delivered-To"]
-                if owner is None:
-                    log.err("Malformed mail, neither To: nor "
-                            "Delivered-To: field")
-                log.msg("Mail owner: %s" % (owner,))
-
-                owner = email.utils.parseaddr(owner)[1]
-                log.msg("%s received a new mail" % (owner,))
-                dpubk = self._users_cdb.getPubKey(owner)
-                duuid = self._users_cdb.queryByAddress(owner)
-                d = DeferredList([dpubk, duuid])
-                d.addCallbacks(self._gather_uuid_pubkey, log.err)
-                d.addCallbacks(self._encrypt_message, log.err,
-                               (owner, mail_data))
-                d.addCallbacks(self._export_message, log.err)
-                d.addCallbacks(self._conditional_remove, log.err,
-                               (filepath,))
-                d.addErrback(log.err)
+        try:
+            if os.path.split(filepath.dirname())[-1]  == "new":
+                log.msg("Processing new mail at %r" % (filepath.path,))
+                with filepath.open("r") as f:
+                    mail_data = f.read()
+                    mail = message_from_string(mail_data)
+                    uuid = self._get_owner(mail)
+                    if uuid is None:
+                        log.msg("Don't know how to deliver mail %r, skipping..." %
+                                filepath.path)
+                        return
+                    log.msg("Mail owner: %s" % (uuid,))
+
+                    if uuid is None:
+                        log.msg("BUG: There was no uuid!")
+                        return
+
+                    d = self._users_cdb.getPubKey(uuid)
+                    d.addCallbacks(self._encrypt_message, log.err,
+                                   (uuid, mail_data))
+                    d.addCallbacks(self._export_message, log.err)
+                    d.addCallbacks(self._conditional_remove, log.err,
+                                   (filepath,))
+                    d.addErrback(log.err)
+        except Exception:
+            log.err()