diff options
| -rw-r--r-- | CHANGELOG | 12 | ||||
| -rw-r--r-- | changes/VERSION_COMPAT | 10 | ||||
| -rw-r--r-- | pkg/requirements.pip | 7 | ||||
| -rw-r--r-- | setup.py | 70 | ||||
| -rw-r--r-- | src/leap/mx/check_recipient_access.py | 8 | ||||
| -rw-r--r-- | src/leap/mx/couchdbhelper.py | 74 | ||||
| -rw-r--r-- | src/leap/mx/mail_receiver.py | 213 |
7 files changed, 266 insertions, 128 deletions
@@ -1,3 +1,15 @@ +0.3.3 Nov 1: + o Fix return codes for check recipient access. Fixes #3356. + o Improve logging in general and support possible unicode parameters + without breaking. + o Try to figure out the encoding of an email first by looking into + its header, if that fails then by using chardet. + o Support more than utf8 encodings for emails. + o Add support for receiving mailing list mails. + o Use the uuid that alias_resolver returned and postfix added to the + mail headers, which improves performance. + o Look for public keys based on uuid instead of mail address. + 0.3.2 Sep 6: o Keep file watcher in memory to prevent losing file events. o Properly save the incoming mail as a doc in couch. diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT new file mode 100644 index 0000000..cc00ecf --- /dev/null +++ b/changes/VERSION_COMPAT @@ -0,0 +1,10 @@ +################################################# +# This file keeps track of the recent changes +# introduced in internal leap dependencies. +# Add your changes here so we can properly update +# requirements.pip during the release process. +# (leave header when resetting) +################################################# +# +# BEGIN DEPENDENCY LIST ------------------------- +# leap.foo.bar>=x.y.z diff --git a/pkg/requirements.pip b/pkg/requirements.pip index d5db275..4242ad4 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -8,7 +8,8 @@ paisley>=0.3.1 # for the time being. couchdb -## XXX change me to whatever you name the package in pypi -python-gnupg>=0.3.0 +leap.common>=0.3.5 leap.soledad.common>=0.3.0 -leap.keymanager>=0.2.0 +leap.keymanager>=0.3.4 + +cchardet # we fallback to chardet if this is not available, but it's preferred @@ -18,6 +18,7 @@ setup file for leap.mx """ import os +import re from setuptools import setup, find_packages import versioneer @@ -43,6 +44,68 @@ trove_classifiers = [ 'Topic :: Security :: Cryptography', ] +DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mx/' + 'archive/%s.tar.gz') +_versions = versioneer.get_versions() +VERSION = _versions['version'] +VERSION_FULL = _versions['full'] +DOWNLOAD_URL = "" + +# get the short version for the download url +_version_short = re.findall('\d+\.\d+\.\d+', VERSION) +if len(_version_short) > 0: + VERSION_SHORT = _version_short[0] + DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT + +cmdclass = versioneer.get_cmdclass() + + +from setuptools import Command + + +class freeze_debianver(Command): + """ + Freezes the version in a debian branch. + To be used after merging the development branch onto the debian one. + """ + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + proceed = str(raw_input( + "This will overwrite the file _version.py. Continue? [y/N] ")) + if proceed != "y": + print("He. You scared. Aborting.") + return + template = r""" +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.7+) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +version_version = '{version}' +version_full = '{version_full}' +""" + templatefun = r""" + +def get_versions(default={}, verbose=False): + return {'version': version_version, 'full': version_full} +""" + subst_template = template.format( + version=VERSION_SHORT, + version_full=VERSION_FULL) + templatefun + with open(versioneer.versionfile_source, 'w') as f: + f.write(subst_template) + + +cmdclass["freeze_debianver"] = freeze_debianver + if os.environ.get("VIRTUAL_ENV", None): data_files = None else: @@ -54,12 +117,15 @@ else: ("/etc/init.d/", ["pkg/leap_mx"])] setup( name='leap.mx', - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), + version=VERSION, + cmdclass=cmdclass, url="http://github.com/leapcode/leap_mx", + download_url=DOWNLOAD_URL, license='AGPLv3+', author='The LEAP Encryption Access Project', author_email='info@leap.se', + maintainer='Kali Kaneko', + maintainer_email='kali@leap.se', description=("An asynchronous, transparently-encrypting remailer " "for the LEAP platform"), long_description=( diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py index 0520c7c..b80ccfd 100644 --- a/src/leap/mx/check_recipient_access.py +++ b/src/leap/mx/check_recipient_access.py @@ -29,11 +29,13 @@ from leap.mx.alias_resolver import AliasResolverFactory class LEAPPostFixTCPMapserverAccess(postfix.PostfixTCPMapServer): def _cbGot(self, value): + # For more info, see: + # http://www.postfix.org/tcp_table.5.html + # http://www.postfix.org/access.5.html if value is None: - self.sendCode(500, postfix.quote("NOT FOUND SORRY")) + self.sendCode(500, postfix.quote("REJECT")) else: - # We do not send the value in this case - self.sendCode(200) + self.sendCode(200, postfix.quote("OK")) class CheckRecipientAccessFactory(AliasResolverFactory): diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index 147e6f9..41604ba 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -29,7 +29,6 @@ except ImportError: print "for instructions on getting required dependencies." try: - from twisted.internet import defer from twisted.python import log except ImportError: print "This software requires Twisted. Please see the README file" @@ -49,16 +48,16 @@ class ConnectedCouchDB(client.CouchDB): """ Connect to a CouchDB instance. - @param host: A hostname string for the CouchDB server. - @type host: str - @param port: The port of the CouchDB server. - @type port: int - @param dbName: (optional) The default database to bind queries to. - @type dbName: str - @param username: (optional) The username for authorization. - @type username: str - @param str password: (optional) The password for authorization. - @type password: str + :param host: A hostname string for the CouchDB server. + :type host: str + :param port: The port of the CouchDB server. + :type port: int + :param dbName: (optional) The default database to bind queries to. + :type dbName: str + :param username: (optional) The username for authorization. + :type username: str + :param str password: (optional) The password for authorization. + :type password: str """ client.CouchDB.__init__(self, host, @@ -78,8 +77,8 @@ class ConnectedCouchDB(client.CouchDB): """ Callback for listDB that prints the available databases - @param data: response from the listDB command - @type data: array + :param data: response from the listDB command + :type data: array """ log.msg("Available databases:") for database in data: @@ -101,10 +100,10 @@ class ConnectedCouchDB(client.CouchDB): """ Check to see if a particular email or alias exists. - @param alias: A string representing the email or alias to check. - @type alias: str - @return: a deferred for this query - @rtype twisted.defer.Deferred + :param alias: A string representing the email or alias to check. + :type alias: str + :return: a deferred for this query + :rtype twisted.defer.Deferred """ assert isinstance(address, (str, unicode)), "Email or alias queries must be string" @@ -124,12 +123,12 @@ class ConnectedCouchDB(client.CouchDB): """ Parses the result of the by_address query and gets the uuid - @param address: alias looked up - @type address: string - @param result: result dictionary - @type result: dict - @return: The uuid for alias if available - @rtype: str + :param address: alias looked up + :type address: string + :param result: result dictionary + :type result: dict + :return: The uuid for alias if available + :rtype: str """ for row in result["rows"]: if row["key"] == address: @@ -140,20 +139,39 @@ class ConnectedCouchDB(client.CouchDB): return uuid return None - def getPubKey(self, address): + def getPubKey(self, uuid): + """ + Returns a deferred that will return the pubkey for the uuid provided + + :param uuid: uuid for the user to query + :type uuid: str + + :rtype: Deferred + """ d = self.openView(docId="Identity", viewId="pgp_key_by_email/", - key=address, + user_id=uuid, reduce=False, include_docs=True) - d.addCallbacks(partial(self._get_pgp_key, address), log.err) + d.addCallbacks(partial(self._get_pgp_key, uuid), log.err) return d - def _get_pgp_key(self, address, result): + def _get_pgp_key(self, uuid, result): + """ + Callback used to filter the correct pubkey from the result of + the query to the couchdb + + :param uuid: uuid for the user that was queried + :type uuid: str + :param result: result dictionary for the db query + :type result: dict + + :rtype: str or None + """ for row in result["rows"]: - if row["key"] == address: + if row["doc"]["user_id"] == uuid: return row["value"] return None diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 5875034..8fcadce 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -25,14 +25,20 @@ import uuid as pyuuid import json import email.utils +import socket + +try: + import cchardet as chardet +except ImportError: + import chardet from email import message_from_string from twisted.application.service import Service from twisted.internet import inotify -from twisted.internet.defer import DeferredList from twisted.python import filepath, log +from leap.common.mail import get_email_charset from leap.soledad.common.document import SoledadDocument from leap.soledad.common.crypto import ( EncryptionSchemes, @@ -54,19 +60,20 @@ class MailReceiver(Service): """ Constructor - @param mail_couch_url: URL prefix for the couchdb where mail + :param mail_couch_url: URL prefix for the couchdb where mail should be stored - @type mail_couch_url: str - @param users_cdb: CouchDB instance from where to get the uuid + :type mail_couch_url: str + :param users_cdb: CouchDB instance from where to get the uuid and pubkey for a user - @type users_cdb: ConnectedCouchDB - @param directories: list of directories to monitor - @type directories: list of tuples (path: str, recursive: bool) + :type users_cdb: ConnectedCouchDB + :param directories: list of directories to monitor + :type directories: list of tuples (path: str, recursive: bool) """ # Service doesn't define an __init__ self._mail_couch_url = mail_couch_url self._users_cdb = users_cdb self._directories = directories + self._domain = socket.gethostbyaddr(socket.gethostname())[0] def startService(self): """ @@ -79,56 +86,49 @@ class MailReceiver(Service): mask = inotify.IN_CREATE for directory, recursive in self._directories: - log.msg("Watching %s --- Recursive: %s" % (directory, recursive)) + log.msg("Watching %r --- Recursive: %r" % (directory, recursive)) self.wm.watch(filepath.FilePath(directory), mask, callbacks=[self._process_incoming_email], recursive=recursive) - def _gather_uuid_pubkey(self, results): - if len(results) < 2: - return None, None - - # DeferredList results are structured like this: - # [ (succeeded, pubkey), (succeeded, uuid) ] - # succeeded is a bool value that specifies if the - # corresponding callback succeeded - pubkey_res, uuid_res = results - - pubkey = pubkey_res[1] if pubkey_res[0] else None - uuid = uuid_res[1] if uuid_res[0] else None - - return uuid, pubkey - - def _encrypt_message(self, uuid_pubkey, address, message): + def _encrypt_message(self, pubkey, uuid, message): """ - Given a UUID, a public key, address and a message, it encrypts - the message to that public key. + Given a UUID, a public key and a message, it encrypts the + message to that public key. The address is needed in order to build the OpenPGPKey object. - @param uuid_pubkey: tuple that holds the uuid and the public - key as it is returned by the previous call in the chain - @type uuid_pubkey: tuple (str, str) - @param address: mail address for this message - @type address: str - @param message: message contents - @type message: str + :param uuid_pubkey: tuple that holds the uuid and the public + key as it is returned by the previous call in the + chain + :type uuid_pubkey: tuple (str, str) + :param message: message contents + :type message: str - @return: uuid, doc to sync with Soledad - @rtype: tuple(str, SoledadDocument) + :return: uuid, doc to sync with Soledad or None, None if + something went wrong. + :rtype: tuple(str, SoledadDocument) """ - uuid, pubkey = uuid_pubkey + if uuid is None or pubkey is None or len(pubkey) == 0: + log.msg("_encrypt_message: Something went wrong, here's all " + "I know: %r | %r" % (uuid, pubkey)) + return None, None + log.msg("Encrypting message to %s's pubkey" % (uuid,)) - log.msg("Pubkey: %s" % (pubkey,)) doc = SoledadDocument(doc_id=str(pyuuid.uuid4())) + encoding = get_email_charset(message, default=None) + if encoding is None: + result = chardet.detect(message) + encoding = result["encoding"] + data = {'incoming': True, 'content': message} if pubkey is None or len(pubkey) == 0: doc.content = { self.INCOMING_KEY: True, ENC_SCHEME_KEY: EncryptionSchemes.NONE, - ENC_JSON_KEY: json.dumps(data) + ENC_JSON_KEY: json.dumps(data, encoding=encoding) } return uuid, doc @@ -136,13 +136,15 @@ class MailReceiver(Service): with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg: gpg.import_keys(pubkey) key = gpg.list_keys().pop() - openpgp_key = openpgp._build_key_from_gpg(address, key, pubkey) + # We don't care about the actual address, so we use a + # dummy one, we just care about the import of the pubkey + openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com", key, pubkey) doc.content = { self.INCOMING_KEY: True, ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, ENC_JSON_KEY: str(gpg.encrypt( - json.dumps(data), + json.dumps(data, encoding=encoding), openpgp_key.fingerprint, symmetric=False)) } @@ -153,20 +155,22 @@ class MailReceiver(Service): """ Given a UUID and a SoledadDocument, it saves it directly in the couchdb that serves as a backend for Soledad, in a db - accessible to the recipient of the mail + accessible to the recipient of the mail. - @param uuid_doc: tuple that holds the UUID and SoledadDocument - @type uuid_doc: tuple(str, SoledadDocument) + :param uuid_doc: tuple that holds the UUID and SoledadDocument + :type uuid_doc: tuple(str, SoledadDocument) - @return: True if it's ok to remove the message, False - otherwise - @rtype: bool + :return: True if it's ok to remove the message, False + otherwise + :rtype: bool """ uuid, doc = uuid_doc - log.msg("Exporting message for %s" % (uuid,)) + if uuid is None or doc is None: + log.msg("_export_message: Something went wrong, here's all " + "I know: %r | %r" % (uuid, doc)) + return False - if uuid is None: - uuid = 0 + log.msg("Exporting message for %s" % (uuid,)) db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,)) db.put_doc(doc) @@ -177,58 +181,83 @@ class MailReceiver(Service): def _conditional_remove(self, do_remove, filepath): """ - Removes the message if do_remove is True + Removes the message if do_remove is True. - @param do_remove: True if the message should be removed, False - otherwise - @type do_remove: bool - @param filepath: path to the mail - @type filepath: twisted.python.filepath.FilePath + :param do_remove: True if the message should be removed, False + otherwise + :type do_remove: bool + :param filepath: path to the mail + :type filepath: twisted.python.filepath.FilePath """ if do_remove: # remove the original mail try: - log.msg("Removing %s" % (filepath.path,)) + log.msg("Removing %r" % (filepath.path,)) filepath.remove() log.msg("Done removing") - except: + except Exception: log.err() + else: + log.msg("Not removing %r" % (filepath.path,)) + + def _get_owner(self, mail): + """ + Given an email, returns the uuid of the owner. + + :param mail: mail to analyze + :type mail: email.message.Message + + :returns: uuid + :rtype: str or None + """ + uuid = None + + delivereds = mail.get_all("Delivered-To") + for to in delivereds: + name, addr = email.utils.parseaddr(to) + parts = addr.split("@") + if len(parts) > 1 and parts[1] == self._domain: + uuid = parts[0] + break + + return uuid def _process_incoming_email(self, otherself, filepath, mask): """ - Callback that processes incoming email - - @param otherself: Watch object for the current callback from - inotify - @type otherself: twisted.internet.inotify._Watch - @param filepath: Path of the file that changed - @type filepath: twisted.python.filepath.FilePath - @param mask: identifier for the type of change that triggered - this callback - @type mask: int + Callback that processes incoming email. + + :param otherself: Watch object for the current callback from + inotify. + :type otherself: twisted.internet.inotify._Watch + :param filepath: Path of the file that changed + :type filepath: twisted.python.filepath.FilePath + :param mask: identifier for the type of change that triggered + this callback + :type mask: int """ - if os.path.split(filepath.dirname())[-1] == "new": - log.msg("Processing new mail at %s" % (filepath.path,)) - with filepath.open("r") as f: - mail_data = f.read() - mail = message_from_string(mail_data) - owner = mail["To"] - if owner is None: # default to Delivered-To - owner = mail["Delivered-To"] - if owner is None: - log.err("Malformed mail, neither To: nor " - "Delivered-To: field") - log.msg("Mail owner: %s" % (owner,)) - - owner = email.utils.parseaddr(owner)[1] - log.msg("%s received a new mail" % (owner,)) - dpubk = self._users_cdb.getPubKey(owner) - duuid = self._users_cdb.queryByAddress(owner) - d = DeferredList([dpubk, duuid]) - d.addCallbacks(self._gather_uuid_pubkey, log.err) - d.addCallbacks(self._encrypt_message, log.err, - (owner, mail_data)) - d.addCallbacks(self._export_message, log.err) - d.addCallbacks(self._conditional_remove, log.err, - (filepath,)) - d.addErrback(log.err) + try: + if os.path.split(filepath.dirname())[-1] == "new": + log.msg("Processing new mail at %r" % (filepath.path,)) + with filepath.open("r") as f: + mail_data = f.read() + mail = message_from_string(mail_data) + uuid = self._get_owner(mail) + if uuid is None: + log.msg("Don't know how to deliver mail %r, skipping..." % + filepath.path) + return + log.msg("Mail owner: %s" % (uuid,)) + + if uuid is None: + log.msg("BUG: There was no uuid!") + return + + d = self._users_cdb.getPubKey(uuid) + d.addCallbacks(self._encrypt_message, log.err, + (uuid, mail_data)) + d.addCallbacks(self._export_message, log.err) + d.addCallbacks(self._conditional_remove, log.err, + (filepath,)) + d.addErrback(log.err) + except Exception: + log.err() |