diff options
| author | Tomas Touceda <chiiph@leap.se> | 2013-05-02 13:44:08 -0300 | 
|---|---|---|
| committer | Tomas Touceda <chiiph@leap.se> | 2013-05-07 14:02:12 -0300 | 
| commit | 860329613b205708dc7a4b04bcaf39b9bd24fec6 (patch) | |
| tree | 9b759eaa8826141efa0ee50eb845e77e91dc8502 | |
| parent | 9042a0f5d7d6cee5bacf8085328a7a29ae99df11 (diff) | |
Several fixes according to the review comments
| -rwxr-xr-x | mx.tac | 85 | ||||
| -rw-r--r-- | src/leap/mx/alias_resolver.py | 28 | ||||
| -rw-r--r-- | src/leap/mx/check_recipient_access.py | 4 | ||||
| -rw-r--r-- | src/leap/mx/couchdbhelper.py | 17 | ||||
| -rw-r--r-- | src/leap/mx/mail_receiver.py | 341 | ||||
| -rwxr-xr-x | start_mx.py | 134 | 
6 files changed, 277 insertions, 332 deletions
| @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# start_mx.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program.  If not, see <http://www.gnu.org/licenses/>. + +import sys +import ConfigParser + +from functools import partial + +from leap.mx import couchdbhelper +from leap.mx.mail_receiver import MailReceiver +from leap.mx.alias_resolver import AliasResolverFactory +from leap.mx.check_recipient_access import CheckRecipientAccessFactory + +try: +    from twisted.application import service, internet +    from twisted.internet import inotify +    from twisted.internet.endpoints import TCP4ServerEndpoint +    from twisted.python import filepath, log +    from twisted.python import usage +except ImportError, ie: +    print "This software requires Twisted>=12.0.2, please see the README for" +    print "help on using virtualenv and pip to obtain requirements." + +config_file = "/etc/leap/mx.conf" + +config = ConfigParser.ConfigParser() +config.read(config_file) + +user = config.get("couchdb", "user") +password = config.get("couchdb", "password") + +server = config.get("couchdb", "server") +port = config.get("couchdb", "port") + +alias_port = config.getint("alias map", "port") +check_recipient_port = config.getint("check recipient", "port") + +cdb = couchdbhelper.ConnectedCouchDB(server, +                                     port=port, +                                     dbName="users", +                                     username=user, +                                     password=password) + + +application = service.Application("LEAP MX") + +# Alias map +alias_map = internet.TCPServer(alias_port, AliasResolverFactory(couchdb=cdb)) +alias_map.setServiceParent(application) + +# Check recipient access +check_recipient = internet.TCPServer(check_recipient_port, +                                     CheckRecipientAccessFactory(couchdb=cdb)) +check_recipient.setServiceParent(application) + +# Mail receiver +mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user, +                                                password, +                                                server, +                                                port) +directories = [] +for section in config.sections(): +    if section in ("couchdb", "alias map", "check recipient"): +        continue +    to_watch = config.get(section, "path") +    recursive = config.getboolean(section, "recursive") +    directories.append([to_watch, recursive]) + +mr = MailReceiver(mail_couch_url_prefix, cdb, directories) +mr.setServiceParent(application) diff --git a/src/leap/mx/alias_resolver.py b/src/leap/mx/alias_resolver.py index 3f93f0d..68c6212 100644 --- a/src/leap/mx/alias_resolver.py +++ b/src/leap/mx/alias_resolver.py @@ -24,18 +24,15 @@ TODO:        controlling concurrent connections and throttling resource consumption.  """ -import logging -  try:      # TODO: we should probably use the system alias somehow      # from twisted.mail      import alias      from twisted.protocols import postfix +    from twisted.python import log  except ImportError:      print "This software requires Twisted. Please see the README file"      print "for instructions on getting required dependencies." -logger = logging.getLogger(__name__) -  class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory):      def __init__(self, couchdb, *args, **kwargs): @@ -46,23 +43,24 @@ class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory):          if isinstance(result, unicode):              result = result.encode("utf8")          if result is None: -            logger.debug("Result not found") +            log.msg("Result not found")          return result      def get(self, key): -        orig_key = key          try: -            logger.debug("Processing key: %s" % (key,)) +            log.msg("Processing key: %s" % (key,))              if key.find("@") == -1: -                logger.debug("Ignoring key since it's not an email address") +                log.msg("Ignoring key since it's not an email address")                  return None              key = key.split("@")[0]              key = key.split("+")[0] -            logger.debug("Final key to query: %s" % (key,)) -        except Exception as e: -            key = orig_key -            logger.exception("%s" % (e,)) -        d = self._cdb.queryByLoginOrAlias(key) -        d.addCallback(self._to_str) -        return d +            log.msg("Final key to query: %s" % (key,)) +            d = self._cdb.queryByLoginOrAlias(key) +            d.addCallback(self._to_str) +            d.addErrback(log.err) +            return d +        except: +            log.err() + +        return None diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py index 3ea6e91..1b44504 100644 --- a/src/leap/mx/check_recipient_access.py +++ b/src/leap/mx/check_recipient_access.py @@ -20,14 +20,10 @@  Classes for resolving postfix recipient access  """ -import logging -  from twisted.protocols import postfix  from leap.mx.alias_resolver import AliasResolverFactory -logger = logging.getLogger(__name__) -  class CheckRecipientAccess(postfix.PostfixTCPMapServer):      def _cbGot(self, value): diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index 2f6b548..7c4c8ce 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -20,8 +20,6 @@ Classes for working with CouchDB or BigCouch instances which store email alias  maps, user UUIDs, and GPG keyIDs.  """ -import logging -  from functools import partial  try: @@ -32,12 +30,11 @@ except ImportError:  try:      from twisted.internet import defer +    from twisted.python import log  except ImportError:      print "This software requires Twisted. Please see the README file"      print "for instructions on getting required dependencies." -logger = logging.getLogger(__name__) -  class ConnectedCouchDB(client.CouchDB):      """ @@ -84,9 +81,9 @@ class ConnectedCouchDB(client.CouchDB):          @param data: response from the listDB command          @type data: array          """ -        logger.msg("Available databases:") +        log.msg("Available databases:")          for database in data: -            logger.msg("  * %s" % (database,)) +            log.msg("  * %s" % (database,))      def createDB(self, dbName):          """ @@ -119,7 +116,7 @@ class ConnectedCouchDB(client.CouchDB):                            reduce=False,                            include_docs=True) -        d.addCallbacks(partial(self._get_uuid, alias), logger.error) +        d.addCallbacks(partial(self._get_uuid, alias), log.err)          return d @@ -138,14 +135,10 @@ class ConnectedCouchDB(client.CouchDB):          for row in result["rows"]:              if row["key"] == alias:                  uuid = row["id"] -                try: -                    self._cache[uuid] = row["doc"]["public_key"] -                except: -                    pass  # no public key for this user +                self._cache[uuid] = row["doc"].get("public_key", None)                  return uuid          return None -      def getPubKey(self, uuid):          pubkey = None          try: diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 00d93ba..2494a21 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -16,190 +16,197 @@  # You should have received a copy of the GNU General Public License  # along with this program.  If not, see <http://www.gnu.org/licenses/>. +""" +MailReceiver service definition +""" +  import os  import uuid as pyuuid -import logging -import argparse -import ConfigParser  import json  from email import message_from_string -from functools import partial - -from twisted.internet import inotify, reactor -from twisted.python import filepath -from leap.mx import couchdbhelper +from twisted.application.service import Service +from twisted.internet import inotify +from twisted.python import filepath, log  from leap.soledad import LeapDocument  from leap.soledad.backends.leap_backend import EncryptionSchemes  from leap.soledad.backends.couch import CouchDatabase  from leap.common.keymanager import openpgp -logger = logging.getLogger(__name__) - - -def _get_pubkey(uuid, cdb): -    logger.debug("Fetching pubkey for %s" % (uuid,)) -    return uuid, cdb.getPubKey(uuid) -def _encrypt_message(uuid_pubkey, address_message): -    uuid, pubkey = uuid_pubkey -    address, message = address_message -    logger.debug("Encrypting message to %s's pubkey" % (uuid,)) -    logger.debug("Pubkey: %s" % (pubkey,)) +class MailReceiver(Service): +    """ +    Service that monitors incoming email and processes it +    """ + +    def __init__(self, mail_couch_url, users_cdb, directories): +        """ +        Constructor + +        @param mail_couch_url: URL prefix for the couchdb where mail +        should be stored +        @type mail_couch_url: str +        @param users_cdb: CouchDB instance from where to get the uuid +        and pubkey for a user +        @type users_cdb: ConnectedCouchDB +        @param directories: list of directories to monitor +        @type directories: list of tuples (path: str, recursive: bool) +        """ +        # Service doesn't define an __init__ +        self._mail_couch_url = mail_couch_url +        self._users_cdb = users_cdb +        self._directories = directories + +    def startService(self): +        """ +        Starts the MailReceiver service +        """ +        Service.startService(self) +        wm = inotify.INotify() +        wm.startReading() + +        mask = inotify.IN_CREATE + +        for directory, recursive in self._directories: +            log.msg("Watching %s --- Recursive: %s" % (directory, recursive)) +            wm.watch(filepath.FilePath(directory), mask, callbacks=[self._process_incoming_email], recursive=recursive) + +    def _get_pubkey(self, uuid): +        """ +        Given a UUID for a user, retrieve its public key + +        @param uuid: UUID for a user +        @type uuid: str + +        @return: uuid, public key +        @rtype: tuple of (str, str) +        """ +        log.msg("Fetching pubkey for %s" % (uuid,)) +        return uuid, self._users_cdb.getPubKey(uuid) + +    def _encrypt_message(self, uuid_pubkey, address, message): +        """ +        Given a UUID, a public key, address and a message, it encrypts +        the message to that public key. +        The address is needed in order to build the OpenPGPKey object. + +        @param uuid_pubkey: tuple that holds the uuid and the public +        key as it is returned by the previous call in the chain +        @type uuid_pubkey: tuple (str, str) +        @param address: mail address for this message +        @type address: str +        @param message: message contents +        @type message: str + +        @return: uuid, doc to sync with Soledad +        @rtype: tuple(str, LeapDocument) +        """ +        uuid, pubkey = uuid_pubkey +        log.msg("Encrypting message to %s's pubkey" % (uuid,)) +        log.msg("Pubkey: %s" % (pubkey,)) + +        doc = LeapDocument(doc_id=str(pyuuid.uuid4())) + +        data = {'incoming': True, 'content': message} + +        if pubkey is None or len(pubkey) == 0: +            doc.content = { +                "_encryption_scheme": EncryptionSchemes.NONE, +                "_unencrypted_json": json.dumps(data) +            } +            return uuid, doc + +        def _ascii_to_openpgp_cb(gpg): +            key = gpg.list_keys().pop() +            return openpgp._build_key_from_gpg(address, key, pubkey) + +        openpgp_key = openpgp._safe_call(_ascii_to_openpgp_cb, pubkey) -    doc = LeapDocument(encryption_scheme=EncryptionSchemes.PUBKEY, -                       doc_id=str(pyuuid.uuid4())) - -    data = {'incoming': True, 'content': message} - -    if pubkey is None or len(pubkey) == 0:          doc.content = { -            "_unencrypted_json": json.dumps(data) +            "_encryption_scheme": EncryptionSchemes.PUBKEY, +            "_encrypted_json": openpgp.encrypt_asym(json.dumps(data), openpgp_key)          } -        return uuid, doc - -    def _ascii_to_openpgp_cb(gpg): -        key = gpg.list_keys().pop() -        return openpgp._build_key_from_gpg(address, key, pubkey) - -    openpgp_key = openpgp._safe_call(_ascii_to_openpgp_cb, pubkey) - -    doc.content = { -        "_encrypted_json": openpgp.encrypt_asym(json.dumps(data), openpgp_key) -    } - -    return uuid, doc - - -def _export_message(uuid_doc, couch_url): -    uuid, doc = uuid_doc -    logger.debug("Exporting message for %s" % (uuid,)) - -    if uuid is None: -        uuid = 0 - -    db = CouchDatabase(couch_url, "user-%s" % (uuid,)) -    db.put_doc(doc) -    logger.debug("Done exporting") - -    return True - - -def _conditional_remove(do_remove, filepath): -    if do_remove: -        # remove the original mail -        try: -            logger.debug("Removing %s" % (filepath.path,)) -            filepath.remove() -            logger.debug("Done removing") -        except Exception as e: -            # TODO: better handle exceptions -            logger.exception("%s" % (e,)) - - -def _process_incoming_email(users_db, mail_couchdb_url_prefix, self, filepath, mask): -    if os.path.split(filepath.dirname())[-1]  == "new": -        logger.debug("Processing new mail at %s" % (filepath.path,)) -        with filepath.open("r") as f: -            mail_data = f.read() -            mail = message_from_string(mail_data) -            owner = mail["To"] -            if owner is None:  # default to Delivered-To -                owner = mail["Delivered-To"] -            owner = owner.split("@")[0] -            owner = owner.split("+")[0] -            logger.debug("Mail owner: %s" % (owner,)) - -            logger.debug("%s received a new mail" % (owner,)) -            d = users_db.queryByLoginOrAlias(owner) -            d.addCallback(_get_pubkey, (users_db)) -            d.addCallback(_encrypt_message, (owner, mail_data)) -            d.addCallback(_export_message, (mail_couchdb_url_prefix)) -            d.addCallback(_conditional_remove, (filepath)) - - -def main(): -    epilog = "Copyright 2012 The LEAP Encryption Access Project" -    parser = argparse.ArgumentParser(description="""LEAP MX Mail receiver""", epilog=epilog) -    parser.add_argument('-d', '--debug', action="store_true", -                        help="Launches the LEAP MX mail receiver with debug output") -    parser.add_argument('-l', '--logfile', metavar="LOG FILE", nargs='?', -                        action="store", dest="log_file", -                        help="Writes the logs to the specified file") -    parser.add_argument('-c', '--config', metavar="CONFIG FILE", nargs='?', -                        action="store", dest="config", -                        help="Where to look for the configuration file. " \ -                            "Default: mail_receiver.cfg") - -    opts, _ = parser.parse_known_args() - -    debug = opts.debug -    config_file = opts.config - -    if debug: -        level = logging.DEBUG -    else: -        level = logging.WARNING - -    if config_file is None: -        config_file = "leap_mx.cfg" - -    logger.setLevel(level) -    console = logging.StreamHandler() -    console.setLevel(level) -    formatter = logging.Formatter( -        '%(asctime)s ' -        '- %(name)s - %(levelname)s - %(message)s') -    console.setFormatter(formatter) -    logger.addHandler(console) +        return uuid, doc -    logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") -    logger.info("    LEAP MX Mail receiver") -    logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") +    def _export_message(self, uuid_doc): +        """ +        Given a UUID and a LeapDocument, it saves it directly in the +        couchdb that serves as a backend for Soledad, in a db +        accessible to the recipient of the mail + +        @param uuid_doc: tuple that holds the UUID and LeapDocument +        @type uuid_doc: tuple(str, LeapDocument) + +        @return: True if it's ok to remove the message, False +        otherwise +        @rtype: bool +        """ +        uuid, doc = uuid_doc +        log.msg("Exporting message for %s" % (uuid,)) + +        if uuid is None: +            uuid = 0 + +        db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,)) +        db.put_doc(doc) + +        log.msg("Done exporting") + +        return True + +    def _conditional_remove(self, do_remove, filepath): +        """ +        Removes the message if do_remove is True + +        @param do_remove: True if the message should be removed, False +        otherwise +        @type do_remove: bool +        @param filepath: path to the mail +        @type filepath: twisted.python.filepath.FilePath +        """ +        if do_remove: +            # remove the original mail +            try: +                log.msg("Removing %s" % (filepath.path,)) +                filepath.remove() +                log.msg("Done removing") +            except: +                log.err() + +    def _process_incoming_email(self, otherself, filepath, mask): +        """ +        Callback that processes incoming email + +        @param otherself: Watch object for the current callback from +        inotify +        @type otherself: twisted.internet.inotify._Watch +        @param filepath: Path of the file that changed +        @type filepath: twisted.python.filepath.FilePath +        @param mask: identifier for the type of change that triggered +        this callback +        @type mask: int +        """ +        if os.path.split(filepath.dirname())[-1]  == "new": +            log.msg("Processing new mail at %s" % (filepath.path,)) +            with filepath.open("r") as f: +                mail_data = f.read() +                mail = message_from_string(mail_data) +                owner = mail["To"] +                if owner is None:  # default to Delivered-To +                    owner = mail["Delivered-To"] +                owner = owner.split("@")[0] +                owner = owner.split("+")[0] +                log.msg("Mail owner: %s" % (owner,)) + +                log.msg("%s received a new mail" % (owner,)) +                d = self._users_cdb.queryByLoginOrAlias(owner) +                d.addCallbacks(self._get_pubkey, log.err) +                d.addCallbacks(self._encrypt_message, log.err, (owner, mail_data)) +                d.addCallbacks(self._export_message, log.err) +                d.addCallbacks(self._conditional_remove, log.err, (filepath,)) +                d.addErrback(log.err) -    logger.info("Reading configuration from %s" % (config_file,)) - -    config = ConfigParser.ConfigParser() -    config.read(config_file) - -    users_user = config.get("couchdb", "users_user") -    users_password = config.get("couchdb", "users_password") - -    mail_user = config.get("couchdb", "mail_user") -    mail_password = config.get("couchdb", "mail_password") - -    server = config.get("couchdb", "server") -    port = config.get("couchdb", "port") - -    wm = inotify.INotify(reactor) -    wm.startReading() - -    mask = inotify.IN_CREATE - -    users_db = couchdbhelper.ConnectedCouchDB(server, -                                              port=port, -                                              dbName="users", -                                              username=users_user, -                                              password=users_password) - -    mail_couch_url_prefix = "http://%s:%s@localhost:%s" % (mail_user, -                                                           mail_password, -                                                           port) - -    incoming_partial = partial(_process_incoming_email, users_db, mail_couch_url_prefix) -    for section in config.sections(): -        if section in ("couchdb"): -            continue -        to_watch = config.get(section, "path") -        recursive = config.getboolean(section, "recursive") -        logger.debug("Watching %s --- Recursive: %s" % (to_watch, recursive)) -        wm.watch(filepath.FilePath(to_watch), mask, callbacks=[incoming_partial], recursive=recursive) - -    reactor.run() - -if __name__ == "__main__": -    main() diff --git a/start_mx.py b/start_mx.py deleted file mode 100755 index bf3dddb..0000000 --- a/start_mx.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -# start_mx.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program.  If not, see <http://www.gnu.org/licenses/>. - -import argparse -import ConfigParser -import logging - -from functools import partial - -from leap.mx import couchdbhelper, mail_receiver -from leap.mx.alias_resolver import AliasResolverFactory -from leap.mx.check_recipient_access import CheckRecipientAccessFactory - -try: -    from twisted.internet import reactor, inotify -    from twisted.internet.endpoints import TCP4ServerEndpoint -    from twisted.python import filepath -except ImportError, ie: -    print "This software requires Twisted>=12.0.2, please see the README for" -    print "help on using virtualenv and pip to obtain requirements." - - -if __name__ == "__main__": -    epilog = "Copyright 2012 The LEAP Encryption Access Project" -    parser = argparse.ArgumentParser(description="""LEAP MX""", -                                     epilog=epilog) -    parser.add_argument('-d', '--debug', action="store_true", -                        help="Launches the LEAP MX mail receiver with debug output") -    parser.add_argument('-l', '--logfile', metavar="LOG FILE", nargs='?', -                        action="store", dest="log_file", -                        help="Writes the logs to the specified file") -    parser.add_argument('-c', '--config', metavar="CONFIG FILE", nargs='?', -                        action="store", dest="config", -                        help="Where to look for the configuration file. " \ -                            "Default: mail_receiver.cfg") - -    opts, _ = parser.parse_known_args() - -    logger = logging.getLogger(name='leap') - -    debug = opts.debug -    config_file = opts.config -    logfile = opts.log_file - -    if debug: -        level = logging.DEBUG -    else: -        level = logging.WARNING - -    if config_file is None: -        config_file = "mx.conf" - -    logger.setLevel(level) -    console = logging.StreamHandler() -    console.setLevel(level) -    formatter = logging.Formatter( -        '%(asctime)s ' -        '- %(name)s - %(levelname)s - %(message)s') -    console.setFormatter(formatter) -    logger.addHandler(console) - -    if logfile is not None: -        logger.debug('Setting logfile to %s ', logfile) -        fileh = logging.FileHandler(logfile) -        fileh.setLevel(logging.DEBUG) -        fileh.setFormatter(formatter) -        logger.addHandler(fileh) - -    logger.info("~~~~~~~~~~~~~~~~~~~") -    logger.info("    LEAP MX") -    logger.info("~~~~~~~~~~~~~~~~~~~") - -    logger.info("Reading configuration from %s" % (config_file,)) - -    config = ConfigParser.ConfigParser() -    config.read(config_file) - -    user = config.get("couchdb", "user") -    password = config.get("couchdb", "password") - -    server = config.get("couchdb", "server") -    port = config.get("couchdb", "port") - -    cdb = couchdbhelper.ConnectedCouchDB(server, -                                         port=port, -                                         dbName="users", -                                         username=user, -                                         password=password) - -    # Mail receiver -    wm = inotify.INotify(reactor) -    wm.startReading() - -    mask = inotify.IN_CREATE - -    mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user, -                                                    password, -                                                    server, -                                                    port) - -    incoming_partial = partial(mail_receiver._process_incoming_email, cdb, mail_couch_url_prefix) -    for section in config.sections(): -        if section in ("couchdb"): -            continue -        to_watch = config.get(section, "path") -        recursive = config.getboolean(section, "recursive") -        logger.debug("Watching %s --- Recursive: %s" % (to_watch, recursive)) -        wm.watch(filepath.FilePath(to_watch), mask, callbacks=[incoming_partial], recursive=recursive) - - -    # Alias map -    alias_endpoint = TCP4ServerEndpoint(reactor, 4242) -    alias_endpoint.listen(AliasResolverFactory(couchdb=cdb)) - -    # Check recipient access -    check_recipient = TCP4ServerEndpoint(reactor, 2244) -    check_recipient.listen(CheckRecipientAccessFactory(couchdb=cdb)) - -    reactor.run() | 
