From 4cdc70715469258a4da9e30bee017b5a881b9ca5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 15 Mar 2013 03:30:00 +0900 Subject: simple setup.py --- setup.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..efceb4c --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# setup.py +# Copyleft (C) 2013 LEAP +""" +setup file for leap.mx +""" +from setuptools import setup, find_packages + +requirements = [ + "twisted", + #... +] + +# XXX add classifiers, docs + +setup( + name='leap.mx', + version='0.0.1', + url='https://leap.se/', + license='', + author="Isis Agora Lovecruft", + author_email=" 0x2cdb8b35", + description=("An asynchronous, transparently-encrypting remailer " + "for the LEAP platform"), + long_description=( + "An asynchronous, transparently-encrypting remailer " + "using BigCouch/CouchDB and PGP/GnuPG, written in Twisted Python." + ), + namespace_packages=["leap"], + package_dir={'': 'src'}, + packages=find_packages('src'), + #test_suite='leap.mx.tests', + install_requires=requirements, +) -- cgit v1.2.3 From b8580daf5ef13e36d20443ac9f8ac21eee0404bb Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 15 Mar 2013 03:30:56 +0900 Subject: declare leap namespace --- src/leap/__init__.py | 18 ++++++------------ src/leap/mx/__init__.py | 5 ++++- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/leap/__init__.py b/src/leap/__init__.py index 8b30f3d..f48ad10 100644 --- a/src/leap/__init__.py +++ b/src/leap/__init__.py @@ -1,12 +1,6 @@ -# -*- encoding: utf-8 -*- -""" -leap/__init__.py ----------------- -Module intialization file for leap. -""" - -from leap.mx.util import version - -__all__ = ['mx'] -__author__ = version.getAuthors() -__version__ = version.getVersion() +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/src/leap/mx/__init__.py b/src/leap/mx/__init__.py index 27fadca..8ef1bc9 100644 --- a/src/leap/mx/__init__.py +++ b/src/leap/mx/__init__.py @@ -2,7 +2,10 @@ """ leap/mx/__init__.py ------------------- -Module intialization file for leap.mx . +Module initialization file for leap.mx . """ +from leap.mx.util import version __all__ = ['alias_resolver', 'couchdb', 'exceptions', 'runner', 'util'] +__author__ = version.getAuthors() +__version__ = version.getVersion() -- cgit v1.2.3 From 41ad788fef8a1add6be528ce8d7ccd17d4f7fe86 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 16 Apr 2013 17:17:05 -0300 Subject: Add stub for the mail_receiver --- src/leap/mx/mail_receiver.py | 132 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 src/leap/mx/mail_receiver.py diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py new file mode 100644 index 0000000..001d476 --- /dev/null +++ b/src/leap/mx/mail_receiver.py @@ -0,0 +1,132 @@ +import os +import pyinotify +import logging +import argparse +import ConfigParser + +from email import message_from_string + +logger = logging.getLogger(name='leap_mx') + + +def _get_uuid(uid, user, password, server): + # TODO: implement! + return "" + + +def _get_pubkey(uuid, user, password, server): + # TODO: implent! + return "" + +def _encrypt_message(pubkey, message): + # TODO: implement! + return message + + +def _export_message(uuid, message, user, password, server): + # TODO: Implement! + return True + +# +# +# +# +# +# + +class EventHandler(pyinotify.ProcessEvent): + def __init__(self, user, password, server, *args, **kwargs): + pyinotify.ProcessEvent.__init__(self, *args, **kwargs) + self._user = user + self._password = password + self._server = server + + def process_IN_CREATE(self, event): + if os.path.split(event.path)[-1] == "new": + logger.debug("Processing new mail at %s" % (event.pathname,)) + with open(event.pathname, "r") as f: + mail_data = f.read() + mail = message_from_string(mail_data) + owner = mail["Delivered-To"] + logger.debug("%s received a new mail" % (owner,)) + # get user uuid + uuid = _get_uuid(owner, self._user, self._password, self._server) + # get the pubkey for uuid + pubkey = _get_pubkey(uuid, self._user, self._password, self._server) + # if the message isn't encrypted already: + # encrypt the message to the pubkey + encrypted = _encrypt_message(pubkey, mail_data) + # save the message in a couchdb + if _export_message(uuid, encrypted, self._user, self._password, self._server): + # remove the original mail + try: + os.remove(event.pathname) + except Exception as e: + # TODO: better handle exceptions + logger.error(e.message()) + +def main(): + epilog = "Copyright 2012 The LEAP Encryption Access Project" + parser = argparse.ArgumentParser(description="""LEAP MX Mail receiver""", epilog=epilog) + parser.add_argument('-d', '--debug', action="store_true", + help="Launches the LEAP MX mail receiver with debug output") + parser.add_argument('-l', '--logfile', metavar="LOG FILE", nargs='?', + action="store", dest="log_file", + help="Writes the logs to the specified file") + parser.add_argument('-c', '--config', metavar="CONFIG FILE", nargs='?', + action="store", dest="config", + help="Where to look for the configuration file. " \ + "Default: mail_receiver.cfg") + + opts, _ = parser.parse_known_args() + + debug = opts.debug + config_file = opts.config + + if debug: + level = logging.DEBUG + else: + level = logging.WARNING + + if config_file is None: + config_file = "mail_receiver.cfg" + + logger.setLevel(level) + console = logging.StreamHandler() + console.setLevel(level) + formatter = logging.Formatter( + '%(asctime)s ' + '- %(name)s - %(levelname)s - %(message)s') + console.setFormatter(formatter) + logger.addHandler(console) + + logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + logger.info(" LEAP MX Mail receiver") + logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + + logger.info("Reading configuration from %s" % (config_file,)) + + config = ConfigParser.ConfigParser() + config.read(config_file) + + user = config.get("couchdb", "user") + password = config.get("couchdb", "password") + server = config.get("couchdb", "server") + + wm = pyinotify.WatchManager() + mask = pyinotify.IN_CREATE + handler = EventHandler(user, password, server) + notifier = pyinotify.Notifier(wm, handler) + + for section in config.sections(): + if section in ("couchdb"): + continue + to_watch = config.get(section, "path") + recursive = config.getboolean(section, "recursive") + logger.debug("Watching %s --- Recursive: %s" % (to_watch, recursive)) + wm.add_watch(to_watch, mask, rec=recursive) + + notifier.loop() + +if __name__ == "__main__": + main() -- cgit v1.2.3 From 37fbc270a68fdc399423fce778e36d7838f3b97e Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 18 Apr 2013 16:19:59 -0300 Subject: Add ipaddr as a dependency --- pkg/mx-requirements.pip | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/mx-requirements.pip b/pkg/mx-requirements.pip index aa8c63b..eda67e9 100644 --- a/pkg/mx-requirements.pip +++ b/pkg/mx-requirements.pip @@ -4,3 +4,4 @@ PyYAML>=3.10 pyxdg>=0.19-5 ## xxx change me to whatever you name the package in pypi #python-gnupg>=0.3.0 +ipaddr \ No newline at end of file -- cgit v1.2.3 From 7270164c0c38c7ff0207271a768c0171371e2586 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 18 Apr 2013 16:20:18 -0300 Subject: Make couchdb work --- src/leap/mx/couchdb.py | 177 ++++++++++++++++++++++++++++--------------------- 1 file changed, 103 insertions(+), 74 deletions(-) diff --git a/src/leap/mx/couchdb.py b/src/leap/mx/couchdb.py index 04cfc4d..277d356 100644 --- a/src/leap/mx/couchdb.py +++ b/src/leap/mx/couchdb.py @@ -1,13 +1,25 @@ # -*- encoding: utf-8 -*- +# couchdb.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ''' couchdb.py ========== Classes for working with CouchDB or BigCouch instances which store email alias maps, user UUIDs, and GPG keyIDs. - -@authors: Isis Agora Lovecruft -@version: 0.0.1-beta -@license: see included LICENSE file ''' try: @@ -22,106 +34,123 @@ except ImportError: print "This software requires Twisted. Please see the README file" print "for instructions on getting required dependencies." +from functools import partial + from leap.mx.util import log class ConnectedCouchDB(client.CouchDB): - """Connect to a CouchDB instance. + """ + Connect to a CouchDB instance. CouchDB document for testing is '_design', and the view is simply a preconfigured set of mapped responses. """ + def __init__(self, host, port=5984, dbName=None, username=None, password=None, *args, **kwargs): """ Connect to a CouchDB instance. - :param str host: A hostname string for the CouchDB server. - :param int port: The port of the CouchDB server. - :param str dbName: (optional) The default database to bind queries to. - :param str username: (optional) The username for authorization. - :param str password: (optional) The password for authorization. - :returns: A :class:`twisted.internet.defer.Deferred` representing the - the client connection to the CouchDB instance. + @param host: A hostname string for the CouchDB server. + @type host: str + @param port: The port of the CouchDB server. + @type port: int + @param dbName: (optional) The default database to bind queries to. + @type dbName: str + @param username: (optional) The username for authorization. + @type username: str + @param str password: (optional) The password for authorization. + @type password: str """ - super(client.CouchDB, self).__init__(host, - port=port, - dbName=dbName, - username=username, - password=password, - *args, **kwargs) + client.CouchDB.__init__(self, + host, + port=port, + dbName=dbName, + username=username, + password=password, + *args, **kwargs) if dbName is None: databases = self.listDB() - log.msg("Available databases: %s" % databases) + databases.addCallback(self._print_databases) + + def _print_databases(self, data): + """ + Callback for listDB that prints the available databases + + @param data: response from the listDB command + @type data: array + """ + log.msg("Available databases:") + for database in data: + log.msg(" * %s" % (database,)) def createDB(self, dbName): - """Overrides ``paisley.client.CouchDB.createDB``.""" + """ + Overrides ``paisley.client.CouchDB.createDB``. + """ pass def deleteDB(self, dbName): - """Overrides ``paisley.client.CouchDB.deleteDB``.""" + """ + Overrides ``paisley.client.CouchDB.deleteDB``. + """ pass - def queryByEmailOrAlias(self, alias, dbDoc="User", - view="by_email_or_alias"): - """Check to see if a particular email or alias exists. + def queryByLoginOrAlias(self, alias): + """ + Check to see if a particular email or alias exists. - :param str alias: A string representing the email or alias to check. - :param str dbDoc: The CouchDB document to open. - :param str view: The view of the CouchDB document to use. + @param alias: A string representing the email or alias to check. + @type alias: str + @return: a deferred for this query + @rtype twisted.defer.Deferred """ assert isinstance(alias, str), "Email or alias queries must be string" - ## Prepend a forward slash, in case we forgot it: - if not alias.startswith('/'): - alias = '/' + alias - - d = self.openDoc(dbDoc) - d.addCallbacks(self.openView, log.err, (view)) - d.addCallbacks(self.get, log.err, (alias)) - d.addCallbacks(self.parseResult, log.err) + d = self.openView(docId="User", + viewId="by_login_or_alias/", + key=alias, + reduce=False) - @d.addCallback - def show_answer(result): - log.msg("Query: %s" % alias) - log.msg("Answer: %s" % alias) + d.addCallbacks(partial(self._get_uuid, alias), log.err) return d - def query(self, uri): - """Query a CouchDB instance that we are connected to. - - :param str uri: A particular URI in the CouchDB, i.e. - "/users/_design/User/_view/by_email_or_alias". + def _get_uuid(self, alias, result): """ - try: - self.checkURI(uri) ## xxx write checkURI() - ## xxx we might be able to use self._parseURI() - except SchemeNotSupported, sns: ## xxx where in paisley is this? - log.exception(sns) ## xxx need log.exception() - - d = self.get(uri) - @d.addCallback - def parse_answer(answer): - return answer - - return answer - - @defer.inlineCallbacks - def listUsersAndEmails(self, limit=1000, reverse=False): - """List all users and email addresses, up to the given limit. - - :param int limit: The number of results to limit the response to. - :param bool reverse: Start at the end of the database mapping. + Parses the result of the by_login_or_alias query and gets the + uuid + + @param alias: alias looked up + @type alias: string + @param result: result dictionary + @type result: dict + @return: The uuid for alias if available + @rtype: str """ - query = "/users/_design/User/_view/by_email_or_alias/?reduce=false" - answer = yield self.query(query, limit=limit, reverse=reverse) - - if answer: - parsed = yield self.parseResult(answer) - if parsed: - log.msg("%s" % parsed) - else: - log.msg("No answer from database, perhaps there are no users.") - else: - log.msg("Problem querying CouchDB instance...") + for row in result["rows"]: + if row["key"] == alias: + return row["id"] + + +if __name__ == "__main__": + from twisted.internet import reactor + cdb = ConnectedCouchDB("localhost", + port=6666, + dbName="users", + username="", + password="") + + d = cdb.queryByLoginOrAlias("test1") + @d.addCallback + def right(result): + print "Should be an actual uuid:", result + + d2 = cdb.queryByLoginOrAlias("asdjaoisdjoiqwjeoi") + @d2.addCallback + def wrong(result): + print "Should be None:", result + + reactor.callLater(5, reactor.stop) + reactor.run() -- cgit v1.2.3 From 6fcb74bda2308d8cd3881c7755bb41020771dc70 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 18 Apr 2013 16:20:43 -0300 Subject: Fix comment since we won't check for encryption before encrypting the email blob --- src/leap/mx/mail_receiver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 001d476..9636591 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -53,8 +53,7 @@ class EventHandler(pyinotify.ProcessEvent): uuid = _get_uuid(owner, self._user, self._password, self._server) # get the pubkey for uuid pubkey = _get_pubkey(uuid, self._user, self._password, self._server) - # if the message isn't encrypted already: - # encrypt the message to the pubkey + # encrypt the message to the pubkey encrypted = _encrypt_message(pubkey, mail_data) # save the message in a couchdb if _export_message(uuid, encrypted, self._user, self._password, self._server): -- cgit v1.2.3 From 6b112dbaed2e778121f1512db1f78f0821ba2aeb Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Fri, 19 Apr 2013 19:05:13 -0300 Subject: More fixes --- src/leap/mx/alias_resolver.py | 429 ++++-------------------------------------- src/leap/mx/couchdb.py | 156 --------------- src/leap/mx/couchdbhelper.py | 172 +++++++++++++++++ src/leap/mx/mail_receiver.py | 157 ++++++++++------ start_mx.py | 217 +++++++++------------ 5 files changed, 401 insertions(+), 730 deletions(-) delete mode 100644 src/leap/mx/couchdb.py create mode 100644 src/leap/mx/couchdbhelper.py diff --git a/src/leap/mx/alias_resolver.py b/src/leap/mx/alias_resolver.py index f71c4d8..eb08c06 100644 --- a/src/leap/mx/alias_resolver.py +++ b/src/leap/mx/alias_resolver.py @@ -1,411 +1,60 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- -''' -alias_resolver.py -================= +# alias_resolver.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" Classes for resolving postfix aliases. -:authors: Isis Agora Lovecruft -:version: 0.0.1-beta -:license: see included LICENSE file -:copyright: (c) 2013 Isis Agora Lovecruft - TODO: - o Look into using twisted.protocols.postfix.policies classes for controlling concurrent connections and throttling resource consumption. -''' +""" -import os -import uuid +import logging try: - from twisted.internet import address, defer, reactor - from twisted.mail import maildir, alias + # TODO: we should probably use the system alias somehow + # from twisted.mail import alias from twisted.protocols import postfix except ImportError: print "This software requires Twisted. Please see the README file" print "for instructions on getting required dependencies." -from leap.mx import couchdb -from leap.mx.util import net, log, config, exceptions - - -def createUUID(alias): - """Creates Universal Unique ID by taking the SHA1 HASH of an email alias: - - >>> uuid.uuid5(uuid.NAMESPACE_URL, "isis@leap.se") - UUID('7194878e-4aea-563f-85a4-4f58519f3c4f') - - TODO: Is there a commonly accepted way to check that an email address - is valid? - - :param str alias: An email address alias. - :returns: A :class:`uuid.UUID` containing attributes specifying the UUID. - """ - return uuid.uuid5(uuid.NAMESPACE_URL, str(alias)) - - -class DatabaseNotConnected(Exception): - """Raised when not currently connected to a database.""" - -class StatusCodes(object): - """The Postfix manual states: - - The request completion status is one of OK, RETRY, NOKEY (lookup failed - because the key was not found), BAD (malformed request) or DENY (the - table is not approved for proxy read or update access). - - In brief, Postfix will send ``get SPACE key NEWLINE``, or - ``put SPACE key NEWLINE`` where ``key`` is an alias or email address. - It expects non-printable ascii characters to be url-encoded, i.e. a - get-request would look like: - - ``get%20isis@leap.se%0A`` - - and in response, Postfix expects an SMTP-like status code and a string - describing the nature of or reason for the response, no longer than - 4096 "characters" (which, due to UTF-8 ubiquity, we'll err on the safe - side and assume that means 4096 bytes.) - - From the Postfix manual on its TCP map protocol - (http://www.postfix.org/tcp_table.5.html): - - 500 SPACE text NEWLINE - In case of a lookup request, the requested data - does not exist. In case of an update request, the - request was rejected. The text describes the - nature of the problem. - - 400 SPACE text NEWLINE - This indicates an error condition. The text - describes the nature of the problem. The client - should retry the request later. - - 200 SPACE text NEWLINE - The request was successful. In the case of a lookup - request, the text contains an encoded version of - the requested data. - - Other SMTP codes: http://www.greenend.org.uk/rjk/tech/smtpreplies.html - - >>> statcodes = StatusCodes() - >>> if : - >>> response_message = statcodes(200) - >>> aliasresolver.tellMTA() - """ - - OK = "OK Others might say 'HELLA AWESOME'...but we're not convinced." - RETRY = "RETRY Server is busy plotting revolution; requests might take a while." - BAD = "BAD bad Leroy Brown, baddest man in the whole...er. Malformed request." - NOKEY = "NOKEY Couldn't find your keys, sorry. Did you check in the sofa?" - DEFER = "DEFER_IF_LOCAL xxx fill me in" - DENY = "DENY no gurlz aloud in teh tree house." - FAIL = "FAIL this belongs on the failblog" - - SMTPCodes = { '200': OK, - '400': RETRY, - '500': BAD, - '550': NOKEY, - '552': DEFER, - '553': DENY, - '554': FAIL, } - - SMTPStrings = { 'OK' 200, - 'RETRY': 400, - 'BAD': 500, - 'NOKEY': 550, - 'DEFER': 552, - 'DENY': 553, - 'FAIL': 554, } - - def __init__(self, status_code=None): - """Construct an SMTP status code generator. - - :type status_code: str or int - :param status_code: (optional) see :func:`StatusCode.get`. - """ - if status_code: - self.get(status_code) - - def get(self, status_code=None) - """Takes an SMTP-like status code and returns an SMTP-like message. - - :type status_code: str or int - :param status_code: The string or integer for the response we want - to give back to the MTA, after looking up an - email address in the local user database. - Can be one of: - * ``OK`` or ``200`` - * ``RETRY`` or ``400`` - * ``BAD`` or ``500`` - * ``NOKEY`` or ``550`` - * ``DEFER`` or ``552`` - * ``DENY`` or ``553`` - * ``FAIL`` or ``554`` - """ - if status_code: - if isinstance(status_code, str): - if status_code.upper() in self.SMTPStrings.keys(): - return self.SMTPStrings[status_code], getattr( - self, status_code.upper(), '') - else: - return 500, self.FAIL - elif isinstance(status_code, int): - for k, v in self.SMTPCodes.items(): - ## we want to return None if it's 500 - if k == str(status_code) and k != '500': - return status_code, v - log.debug("%s" % self.NOKEY) - return None, '' - - -class AliasResolver(postfix.PostfixTCPMapServer): - """Resolve postfix aliases, similarly to using "$ postmap -q ". - - This class starts a simple LineReceiver server which listens for a string - specifying an alias ``key`` to look up, which will be used to query the - local user database. You can test it with: - - $ ./alias_resolver.py & - $ /usr/bin/postmap -q tcp:localhost:1347 - - Resources: - http://www.postfix.org/proxymap.8.html - https://www.iana.org/assignments/smtp-enhanced-status-codes/ - """ - - virtual_transport = '@example.com' - use_virtual_transport = False - - def __init__(self, *args, **kwargs): - """Create a server which listens for Postfix aliases to resolve. - - :param int timeout: Number of seconds to wait for a response. - :param str delimiter: The delimiter to use for the EOL on responses. - (Default: '\n') - """ - super(postfix.PostfixTCPMapServer, self).__init__(*args, **kwargs) - self.status_codes = StatusCodes() - - def sendCode(self, code, message=None): - """Send an SMTP-like code with a message. - - :type code: str or int - :param code: The status code to send, see - ``alias_resolver.StatusCodes``. - """ - try: - assert isinstance(code, int), "status code must be type int" - except AssertionError as ae: - log.err(ae.message) - self.sendLine('500 internal server error: %s' % ae.message) - - msg = self.status_codes.get(code) - if message is not None and isinstance(message, str): - msg += (" " + message) - self.sendLine('%3.3d %s' % (code, message or '')) - - def do_get(self, key): - """Make a query to resolve an alias.""" - if key is None: - self.sendCode(500) - log.warn("Command 'get' takes one parameter.") - else: - d = defer.maybeDeferred(self.factory.get, key) - d.addCallbacks(self._cbGot, self._cbNot) - d.addErrback(log.err) - - @defer.inlineCallbacks - def do_put(self, keyAndValue): - """Add a key and value to the database, provided it does not exist. - - :param str keyAndValue: An alias and email address, separated by a - space, i.e. ``"isis isis@leap.se"``. - """ - if keyAndValue is None: - self.sendCode(500) - log.warn("Command 'put' takes two parameters.") - else: - try: - key, value = keyAndValue.split(None, 1) - except ValueError: - self.sendCode(500) - log.warn("Command 'put' takes two parameters.") - else: - alreadyThere = yield self.do_query(key) - if alreadyThere is None: - d = defer.maybeDeferred(self.factory.put, key, value) - d.addCallbacks(self._cbPut, self._cbPout) - d.addCallbacks(log.err) - else: - self.sendCode(553) - - @defer.inlineCallbacks - def do_delete(self, key): - """Delete an alias from the CouchDB. - - xxx I'm not sure if implementing this would be a good idea... - - :param str key: An email address to delete from the CouchDB. - """ - raise NotImplemented - - def check_recipient_access(self, key): - """Make a query to the CouchDB to resolve an alias. - - If the ``key`` is an email address which the CouchDB has information - for that account, we should respond to Postfix with an '200%20\n". - - :param str key: An email address to look up in the CouchDB. - """ - return self.do_get(key) - - def virtual_alias_map(self, key): - """Get the Universal Unique ID for the alias address. If - virtual_transport is True, then suffix the UUID with a domain. - - xxx I don't think we actually need couchdb for this, the UUID is an - identifier, not an authenticator. And the SHA1 should always be the - same, so unless it's considered to expensive to compute (less than - querying a database, I would presume), it seems silly to do this. - - Instead, we should query CouchDB with the UUID to get the GPG keyid. - - xxx Or are we supposed to query Soledad for this? - - :param str key: An email address to look up in the CouchDB. - :returns: The UUID of the user. - """ - ## xxx need email address parser - userid = createUUID(key) - - if self.use_virtual_transport \ - and isinstance(self.virtual_transport, str): - return userid.get_urn() + self.virtual_transport - else: - return userid.get_urn() +logger = logging.getLogger(__name__) class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory): - """A Factory for creating :class:`AliasResolver` servers, which handles - inputs and outputs, and keeps an in-memory mapping of Postfix aliases in - the form of a dictionary. - - >>> from leap.mx import alias_resolver - >>> aliasResolverFactory = alias_resolver.AliasResolver( - ... data={'isis': 'isis@leap.se', - ... 'drebs': 'drebs@leap.se', - ... 'elijah': 'elijah@leap.se',}) - >>> aliasResolver = aliasResolverFactory.buildProtocol() - >>> aliasResolver.check_recipient_access('isis') - """ - protocol = AliasResolver - database = couchdb.ConnectedCouchDB - - def __init__(self, addr='127.0.0.1', port=4242, timeout=120, - data=None, virtual_transport=None, use_virtual_transport=False, - couch_host=None, couch_port=None, couch_dbname='users', - couch_username=None, couch_password=None): - """Create a Factory which returns :class:`AliasResolver` servers. - - :param str addr: A string giving the IP address of this server, for - talking to postfix. Default: '127.0.0.1' - :param int port: An integer that specifies the port number that this - server should listen and respond on, for talking to - Postfix. on. Default: 4242 - :param int timeout: An integer specifying the number of seconds to wait - until we should time out. Default: 120 - :param dict data: A dict to use to initialise or update the alias - mapping. - :param str virtual_transport: The domain portion of an email address - to suffix the UUID responses of - ``AliasResolver.virtual_alias_map`` with. - :param bool use_virtual_transport: If True, suffix UUIDs with the - ``virtual_transport`` string. + def __init__(self, couchdb, *args, **kwargs): + postfix.PostfixTCPMapDeferringDictServerFactory.__init__(self, *args, **kwargs) + self._cdb = couchdb - :param str couch_host: The IP address of the CouchDB server to query. - :param int couch_port: The port of the CouchDB server to query. - :param str couch_dbname: The database in the CouchDB to bind to. - :param str couch_username: The username for authenticating to the - CouchDB. - :param str couch_password: The password for authentication. - """ - super(postfix.PostfixTCPMapDeferringDictServerFactory, - self).__init__(data=data) - self.timeout = timeout - self.virtual_transport = virtual_transport - self.use_virtual_transport = use_virtual_transport - self.noisy = True if config.advanced.noisy else False - - if couch_port is None: - couch_port = 5984 - if couch_dbname is None: - couch_dbname = 'users' - self.database_connected = False - if couch_host is not None: - self.couch = self.connectDatabase(couch_host, couch_port, - couch_dbname, couch_username, - couch_password) + def _to_str(self, result): + if isinstance(result, unicode): + result = result.encode("utf8") + return result + def get(self, key): + orig_key = key try: - assert isinstance(port, int), "Port number must be an integer" - assert isinstance(timeout, int), "Timeout must be an integer" - except AssertionError, ae: - raise SystemExit(ae.message) - - if net.checkIPaddress(addr): - self.addr = address._IPAddress('TCP', addr, int(port)) - else: - log.msg("Using default address: 127.0.0.1:%s" % port) - self.addr = address._IPAddress('TCP', '127.0.0.1', int(port)) - - log.msg("To configure Postfix to query this alias_resolver,") - log.msg("you should do:") - log.msg(" $ postconf -e 'check_recipient_access = tcp:%s:%d" - % (addr, port)) - - def buildProtocol(self): - """Create an instance of the :class:`AliasResolver` server.""" - proto = self.protocol() - proto.timeout = self.timeout - proto.virtual_transport = self.virtual_transport - proto.use_virtual_transport = self.use_virtual_transport - proto.factory = self - return proto - - def _cb_connectDatabase(self): - self.database_connected = True - - def connectDatabase(self, couch_host, couch_port=None, couch_dbname=None, - couch_username=None, couch_password=None): - """Connect to the CouchDB instance.""" - if not self.database_connected: - d = self.database(couch_host, couch_port, dbName=couch_dbname, - username=couch_username, password=couch_password) - d.addCallback(self._cb_connectDatabase) - d.addErrback(log.err) - return d - else: - return self.couch ## xxx are we sure we only want one connection? - - def get(self, key, **kwargs): - """Query the CouchDB for a user's info. - - :param str key: The alias to look up. Should be either an email address - or a username. (xxx do we want to also support lookups - by UUID?) - """ - if self.database_connected: - return self.couch.queryByEmailOrAlias(key) - else: - raise DatabaseNotConnected("Must be connected to a database.") - - def put(self, key, **kwargs): - """Add an alias to the CouchDB database.""" - raise NotImplemented - - -if __name__ == "__main__": - - print "To test alias_resolver.py, please use /test/test_alias_resolver.py" + key = key.split("@")[0] + key = key.split("+")[0] + except Exception as e: + key = orig_key + logger.exception("%s" % (e,)) + d = self._cdb.queryByLoginOrAlias(key) + d.addCallback(self._to_str) + return d diff --git a/src/leap/mx/couchdb.py b/src/leap/mx/couchdb.py deleted file mode 100644 index 277d356..0000000 --- a/src/leap/mx/couchdb.py +++ /dev/null @@ -1,156 +0,0 @@ -# -*- encoding: utf-8 -*- -# couchdb.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -''' -couchdb.py -========== -Classes for working with CouchDB or BigCouch instances which store email alias -maps, user UUIDs, and GPG keyIDs. -''' - -try: - from paisley import client -except ImportError: - print "This software requires paisley. Please see the README file" - print "for instructions on getting required dependencies." - -try: - from twisted.internet import defer -except ImportError: - print "This software requires Twisted. Please see the README file" - print "for instructions on getting required dependencies." - -from functools import partial - -from leap.mx.util import log - - -class ConnectedCouchDB(client.CouchDB): - """ - Connect to a CouchDB instance. - - CouchDB document for testing is '_design', and the view is simply - a preconfigured set of mapped responses. - """ - - def __init__(self, host, port=5984, dbName=None, username=None, - password=None, *args, **kwargs): - """ - Connect to a CouchDB instance. - - @param host: A hostname string for the CouchDB server. - @type host: str - @param port: The port of the CouchDB server. - @type port: int - @param dbName: (optional) The default database to bind queries to. - @type dbName: str - @param username: (optional) The username for authorization. - @type username: str - @param str password: (optional) The password for authorization. - @type password: str - """ - client.CouchDB.__init__(self, - host, - port=port, - dbName=dbName, - username=username, - password=password, - *args, **kwargs) - if dbName is None: - databases = self.listDB() - databases.addCallback(self._print_databases) - - def _print_databases(self, data): - """ - Callback for listDB that prints the available databases - - @param data: response from the listDB command - @type data: array - """ - log.msg("Available databases:") - for database in data: - log.msg(" * %s" % (database,)) - - def createDB(self, dbName): - """ - Overrides ``paisley.client.CouchDB.createDB``. - """ - pass - - def deleteDB(self, dbName): - """ - Overrides ``paisley.client.CouchDB.deleteDB``. - """ - pass - - def queryByLoginOrAlias(self, alias): - """ - Check to see if a particular email or alias exists. - - @param alias: A string representing the email or alias to check. - @type alias: str - @return: a deferred for this query - @rtype twisted.defer.Deferred - """ - assert isinstance(alias, str), "Email or alias queries must be string" - - d = self.openView(docId="User", - viewId="by_login_or_alias/", - key=alias, - reduce=False) - - d.addCallbacks(partial(self._get_uuid, alias), log.err) - - return d - - def _get_uuid(self, alias, result): - """ - Parses the result of the by_login_or_alias query and gets the - uuid - - @param alias: alias looked up - @type alias: string - @param result: result dictionary - @type result: dict - @return: The uuid for alias if available - @rtype: str - """ - for row in result["rows"]: - if row["key"] == alias: - return row["id"] - - -if __name__ == "__main__": - from twisted.internet import reactor - cdb = ConnectedCouchDB("localhost", - port=6666, - dbName="users", - username="", - password="") - - d = cdb.queryByLoginOrAlias("test1") - @d.addCallback - def right(result): - print "Should be an actual uuid:", result - - d2 = cdb.queryByLoginOrAlias("asdjaoisdjoiqwjeoi") - @d2.addCallback - def wrong(result): - print "Should be None:", result - - reactor.callLater(5, reactor.stop) - reactor.run() diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py new file mode 100644 index 0000000..cbb087d --- /dev/null +++ b/src/leap/mx/couchdbhelper.py @@ -0,0 +1,172 @@ +# -*- encoding: utf-8 -*- +# couchdb.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Classes for working with CouchDB or BigCouch instances which store email alias +maps, user UUIDs, and GPG keyIDs. +""" + +try: + from paisley import client +except ImportError: + print "This software requires paisley. Please see the README file" + print "for instructions on getting required dependencies." + +try: + from twisted.internet import defer +except ImportError: + print "This software requires Twisted. Please see the README file" + print "for instructions on getting required dependencies." + +from functools import partial + +from leap.mx.util import log + + +class ConnectedCouchDB(client.CouchDB): + """ + Connect to a CouchDB instance. + + CouchDB document for testing is '_design', and the view is simply + a preconfigured set of mapped responses. + """ + + def __init__(self, host, port=5984, dbName=None, username=None, + password=None, *args, **kwargs): + """ + Connect to a CouchDB instance. + + @param host: A hostname string for the CouchDB server. + @type host: str + @param port: The port of the CouchDB server. + @type port: int + @param dbName: (optional) The default database to bind queries to. + @type dbName: str + @param username: (optional) The username for authorization. + @type username: str + @param str password: (optional) The password for authorization. + @type password: str + """ + client.CouchDB.__init__(self, + host, + port=port, + dbName=dbName, + username=username, + password=password, + *args, **kwargs) + + self._cache = {} + + if dbName is None: + databases = self.listDB() + databases.addCallback(self._print_databases) + + def _print_databases(self, data): + """ + Callback for listDB that prints the available databases + + @param data: response from the listDB command + @type data: array + """ + log.msg("Available databases:") + for database in data: + log.msg(" * %s" % (database,)) + + def createDB(self, dbName): + """ + Overrides ``paisley.client.CouchDB.createDB``. + """ + pass + + def deleteDB(self, dbName): + """ + Overrides ``paisley.client.CouchDB.deleteDB``. + """ + pass + + def queryByLoginOrAlias(self, alias): + """ + Check to see if a particular email or alias exists. + + @param alias: A string representing the email or alias to check. + @type alias: str + @return: a deferred for this query + @rtype twisted.defer.Deferred + """ + assert isinstance(alias, str), "Email or alias queries must be string" + + # TODO: Cache results + + d = self.openView(docId="User", + viewId="by_login_or_alias/", + key=alias, + reduce=False) + + d.addCallbacks(partial(self._get_uuid, alias), log.err) + + return d + + def _get_uuid(self, alias, result): + """ + Parses the result of the by_login_or_alias query and gets the + uuid + + @param alias: alias looked up + @type alias: string + @param result: result dictionary + @type result: dict + @return: The uuid for alias if available + @rtype: str + """ + for row in result["rows"]: + if row["key"] == alias: + uuid = row["id"] + self._cache[uuid] = row["value"] + return uuid + return None + + + def getPubKey(self, uuid): + pubkey = None + try: + pubkey = self._cache[uuid] + except: + pass + return pubkey + + +if __name__ == "__main__": + from twisted.internet import reactor + cdb = ConnectedCouchDB("localhost", + port=6666, + dbName="users", + username="", + password="") + + d = cdb.queryByLoginOrAlias("test1") + @d.addCallback + def right(result): + print "Should be an actual uuid:", result + print cdb.getPubKey(result) + + d2 = cdb.queryByLoginOrAlias("asdjaoisdjoiqwjeoi") + @d2.addCallback + def wrong(result): + print "Should be None:", result + + reactor.callLater(5, reactor.stop) + reactor.run() diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 9636591..09200ac 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -1,68 +1,97 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# mail_receiver.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + import os -import pyinotify import logging import argparse import ConfigParser from email import message_from_string +from functools import partial -logger = logging.getLogger(name='leap_mx') +from twisted.internet import inotify, reactor +from twisted.python import filepath +from leap.mx import couchdbhelper +from leap.soledad.backends.couch import CouchDatabase -def _get_uuid(uid, user, password, server): - # TODO: implement! - return "" +logger = logging.getLogger(__name__) -def _get_pubkey(uuid, user, password, server): +def _get_pubkey(uuid): # TODO: implent! - return "" + logger.debug("Fetching pubkey for %s" % (uuid,)) + return uuid, "" -def _encrypt_message(pubkey, message): +def _encrypt_message(uuid_pubkey, message): # TODO: implement! - return message + uuid, pubkey = uuid_pubkey + logger.debug("Encrypting message to %s's pubkey" % (uuid,)) + logger.debug("Pubkey: %s" % (pubkey,)) + + if pubkey is None or len(pubkey) == 0: + # TODO: This is only for testing!! REMOVE! + return uuid, message + + encrypted = "" + + return uuid, encrypted + + +def _export_message(uuid_message, couch_url): + uuid, message = uuid_message + logger.debug("Exporting message for %s" % (uuid,)) + + if uuid is None: + uuid = 0 + db_url = couch_url + '/user-%s' % uuid + db = CouchDatabase.open_database(db_url, create=True) + doc = db.create_doc({'content': str(message)}) -def _export_message(uuid, message, user, password, server): - # TODO: Implement! return True -# -# -# -# -# -# - -class EventHandler(pyinotify.ProcessEvent): - def __init__(self, user, password, server, *args, **kwargs): - pyinotify.ProcessEvent.__init__(self, *args, **kwargs) - self._user = user - self._password = password - self._server = server - - def process_IN_CREATE(self, event): - if os.path.split(event.path)[-1] == "new": - logger.debug("Processing new mail at %s" % (event.pathname,)) - with open(event.pathname, "r") as f: - mail_data = f.read() - mail = message_from_string(mail_data) - owner = mail["Delivered-To"] - logger.debug("%s received a new mail" % (owner,)) - # get user uuid - uuid = _get_uuid(owner, self._user, self._password, self._server) - # get the pubkey for uuid - pubkey = _get_pubkey(uuid, self._user, self._password, self._server) - # encrypt the message to the pubkey - encrypted = _encrypt_message(pubkey, mail_data) - # save the message in a couchdb - if _export_message(uuid, encrypted, self._user, self._password, self._server): - # remove the original mail - try: - os.remove(event.pathname) - except Exception as e: - # TODO: better handle exceptions - logger.error(e.message()) + +def _conditional_remove(do_remove, filepath): + if do_remove: + # remove the original mail + try: + logger.debug("Removing %s" % (filepath.path,)) + filepath.remove() + except Exception as e: + # TODO: better handle exceptions + logger.exception("%s" % (e,)) + + +def _process_incoming_email(users_db, mail_couchdb_url_prefix, self, filepath, mask): + if os.path.split(filepath.dirname())[-1] == "new": + logger.debug("Processing new mail at %s" % (filepath.path,)) + with filepath.open("r") as f: + mail_data = f.read() + mail = message_from_string(mail_data) + owner = mail["Delivered-To"] + logger.debug("%s received a new mail" % (owner,)) + d = users_db.queryByLoginOrAlias(owner) + d.addCallback(_get_pubkey) + d.addCallback(_encrypt_message, (mail_data)) + d.addCallback(_export_message, (mail_couchdb_url_prefix)) + d.addCallback(_conditional_remove, (filepath)) + def main(): epilog = "Copyright 2012 The LEAP Encryption Access Project" @@ -88,7 +117,7 @@ def main(): level = logging.WARNING if config_file is None: - config_file = "mail_receiver.cfg" + config_file = "leap_mx.cfg" logger.setLevel(level) console = logging.StreamHandler() @@ -108,24 +137,40 @@ def main(): config = ConfigParser.ConfigParser() config.read(config_file) - user = config.get("couchdb", "user") - password = config.get("couchdb", "password") + users_user = config.get("couchdb", "users_user") + users_password = config.get("couchdb", "users_password") + + mail_user = config.get("couchdb", "mail_user") + mail_password = config.get("couchdb", "mail_password") + server = config.get("couchdb", "server") + port = config.get("couchdb", "port") + + wm = inotify.INotify(reactor) + wm.startReading() + + mask = inotify.IN_CREATE + + users_db = couchdbhelper.ConnectedCouchDB(server, + port=port, + dbName="users", + username=users_user, + password=users_password) - wm = pyinotify.WatchManager() - mask = pyinotify.IN_CREATE - handler = EventHandler(user, password, server) - notifier = pyinotify.Notifier(wm, handler) + mail_couch_url_prefix = "http://%s:%s@localhost:%s" % (mail_user, + mail_password, + port) + incoming_partial = partial(_process_incoming_email, users_db, mail_couch_url_prefix) for section in config.sections(): if section in ("couchdb"): continue to_watch = config.get(section, "path") recursive = config.getboolean(section, "recursive") logger.debug("Watching %s --- Recursive: %s" % (to_watch, recursive)) - wm.add_watch(to_watch, mask, rec=recursive) + wm.watch(filepath.FilePath(to_watch), mask, callbacks=[incoming_partial], recursive=recursive) - notifier.loop() + reactor.run() if __name__ == "__main__": main() diff --git a/start_mx.py b/start_mx.py index d45209c..d2905c5 100755 --- a/start_mx.py +++ b/start_mx.py @@ -1,146 +1,107 @@ #!/usr/bin/env python -#-*- coding: utf-8 -*- -""" - ____ - | MX |_________________________ - ___|____| An encrypting remailer |________ - | |__________________________| | - | is designed for use on a mail exchange | - | with OpenPGP implementations and Postfix, | - | and is part of the Leap Encryption Access | - | Project platform. | - |___________________________________________| -""" - # authors: Isis Agora Lovecruft, 0x2cdb8b35 - # license: AGPLv3, see included LICENCE file. - # copyright: copyright (c) 2013 Isis Agora Lovecruft - - -from __future__ import print_function -from os import getcwd -from os import path as ospath - +# -*- encoding: utf-8 -*- +# start_mx.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse import sys +import ConfigParser +import logging - -application_name = "leap_mx" - -def __get_dirs__(): - """Get the absolute path of the top-level repository directory.""" - here = getcwd() - base = here.rsplit(application_name, 1)[0] - repo = ospath.join(base, application_name) - leap = ospath.join(repo, 'src') - ours = ospath.join(leap, application_name.replace('_', '/')) - return repo, leap, ours - -## py3k check, snagged from python-gnupg-0.3.2 by Vinay Sajip -try: - unicode - _py3k = False -except NameError: - _py3k = True - -## Set the $PYTHONPATH: -repo, leap, ours = __get_dirs__() -sys.path[:] = map(ospath.abspath, sys.path) -sys.path.insert(0, leap) - -## Now we should be able to import ourselves without installation: try: - from leap.mx import runner - from leap.mx.util import config, log, version + from leap.mx import couchdb + from leap.mx.alias_resolver import AliasResolverFactory except ImportError, ie: - print("%s \nExiting... \n" % ie.message) + print "%s \nExiting... \n" % ie.message sys.exit(1) try: - from twisted.python import usage, runtime, failure - from twisted.python.util import spewer + from twisted.internet import reactor + from twisted.internet.endpoints import TCP4ServerEndpoint except ImportError, ie: - print("This software requires Twisted>=12.0.2, please see the README for") - print("help on using virtualenv and pip to obtain requirements.") + print "This software requires Twisted>=12.0.2, please see the README for" + print "help on using virtualenv and pip to obtain requirements." +logger = logging.getLogger(__name__) -class MXOptions(usage.Options): - """Command line options for leap_mx.""" - optParameters = [ - ['config', 'c', 'mx.conf', 'Config file to use']] - optFlags = [ - ['all-tests', 'a', 'Run all unittests'], - ['verbose', 'v', 'Increase logging verbosity']] +if __name__ == "__main__": + epilog = "Copyright 2012 The LEAP Encryption Access Project" + parser = argparse.ArgumentParser(description="""LEAP MX""", + epilog=epilog) + parser.add_argument('-d', '--debug', action="store_true", + help="Launches the LEAP MX mail receiver with debug output") + parser.add_argument('-l', '--logfile', metavar="LOG FILE", nargs='?', + action="store", dest="log_file", + help="Writes the logs to the specified file") + parser.add_argument('-c', '--config', metavar="CONFIG FILE", nargs='?', + action="store", dest="config", + help="Where to look for the configuration file. " \ + "Default: mail_receiver.cfg") + + opts, _ = parser.parse_known_args() + + debug = opts.debug + config_file = opts.config + + if debug: + level = logging.DEBUG + else: + level = logging.WARNING - def opt_version(self): - """Print leap_mx version and exit.""" - print("Authors: %s" % version.getAuthors()) - print("Licence: AGPLv3, see included LICENSE file") - print("Copyright: © 2013 Isis Lovecruft, see included COPYLEFT file") - print("Version: %s" % version.getVersion()) - sys.exit(0) + if config_file is None: + config_file = "mx.conf" - def opt_spewer(self): - """Print *all of the things*. Useful for debugging.""" - sys.settrace(spewer) + logger.setLevel(level) + console = logging.StreamHandler() + console.setLevel(level) + formatter = logging.Formatter( + '%(asctime)s ' + '- %(name)s - %(levelname)s - %(message)s') + console.setFormatter(formatter) + logger.addHandler(console) - def parseArgs(self): - """Called with the remaining unrecognised commandline options.""" - log.warn("Couldn't recognise option: %s" % self) + logger.info("~~~~~~~~~~~~~~~~~~~") + logger.info(" LEAP MX") + logger.info("~~~~~~~~~~~~~~~~~~~") + logger.info("Reading configuration from %s" % (config_file,)) -if __name__ == "__main__": - dependency_check = runner.CheckRequirements(version.getPackageName(), - version.getPipfile()) - ## the following trickery is for printing the module docstring - ## *before* the options help, and printing it only once: - import __main__ - print("%s" % __main__.__doc__) - __main__.__doc__ = (""" -Example Usage: - $ start_mx.py --config="./my-mx.conf" --spewer -""") - - mx_options = MXOptions() - if len(sys.argv) <= 1: - mx_options.opt_help() - sys.exit(0) - try: - mx_options.parseOptions() - except usage.UsageError, ue: - print("%s" % ue.message) - sys.exit(1) - options = mx_options.opts - - ## Get the config settings: - config.filename = options['config'] - config.loadConfig() - - if config.basic.enable_logfile: - ## Log to file: - logfilename = config.basic.logfile - logfilepath = ospath.join(repo, 'logs') - log.start(logfilename, logfilepath) - else: - ## Otherwise just log to stdout: - log.start() + config = ConfigParser.ConfigParser() + config.read(config_file) - log.msg("Testing logging functionality") - if runtime.platform.supportsThreads(): - thread_support = "with thread support." - else: - thread_support = "without thread support." - log.debug("Running %s, with Python %s on %s platform %s" - % (application_name, runtime.shortPythonVersion(), - runtime.platform.getType(), thread_support)) - - if options['verbose']: - config.basic.debug = True - failure.traceupLength = 7 - failure.startDebugMode() - - if options['all-tests']: - from leap.mx import tests - tests.run() - else: - mx_options.getUsage() - sys.exit(1) + users_user = config.get("couchdb", "users_user") + users_password = config.get("couchdb", "users_password") + + mail_user = config.get("couchdb", "mail_user") + mail_password = config.get("couchdb", "mail_password") + + server = config.get("couchdb", "server") + port = config.get("couchdb", "port") + + cdb = couchdb.ConnectedCouchDB(server, + port=port, + dbName="users", + username=users_user, + password=users_password) + + # TODO: use the couchdb for mail + + # TODO: make the listening ports configurable + alias_endpoint = TCP4ServerEndpoint(reactor, 4242) + alias_endpoint.listen(AliasResolverFactory(couchdb=cdb)) + + reactor.run() -- cgit v1.2.3 From b322a390af83addcf9768ae59d31c31a3229b49d Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 23 Apr 2013 17:02:55 -0300 Subject: Add pubkey retrieving --- src/leap/mx/alias_resolver.py | 4 ++++ src/leap/mx/couchdbhelper.py | 6 ++++-- start_mx.py | 16 ++++++++-------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/leap/mx/alias_resolver.py b/src/leap/mx/alias_resolver.py index eb08c06..8155ff9 100644 --- a/src/leap/mx/alias_resolver.py +++ b/src/leap/mx/alias_resolver.py @@ -45,13 +45,17 @@ class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory): def _to_str(self, result): if isinstance(result, unicode): result = result.encode("utf8") + if result is None: + logger.debug("Result not found") return result def get(self, key): orig_key = key try: + logger.debug("Processing key: %s" % (key,)) key = key.split("@")[0] key = key.split("+")[0] + logger.debug("Final key to query: %s" % (key,)) except Exception as e: key = orig_key logger.exception("%s" % (e,)) diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index cbb087d..4e76be3 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -114,7 +114,8 @@ class ConnectedCouchDB(client.CouchDB): d = self.openView(docId="User", viewId="by_login_or_alias/", key=alias, - reduce=False) + reduce=False, + include_docs=True) d.addCallbacks(partial(self._get_uuid, alias), log.err) @@ -135,7 +136,7 @@ class ConnectedCouchDB(client.CouchDB): for row in result["rows"]: if row["key"] == alias: uuid = row["id"] - self._cache[uuid] = row["value"] + self._cache[uuid] = row["doc"]["public_key"] return uuid return None @@ -161,6 +162,7 @@ if __name__ == "__main__": @d.addCallback def right(result): print "Should be an actual uuid:", result + print "Public Key:" print cdb.getPubKey(result) d2 = cdb.queryByLoginOrAlias("asdjaoisdjoiqwjeoi") diff --git a/start_mx.py b/start_mx.py index d2905c5..c802852 100755 --- a/start_mx.py +++ b/start_mx.py @@ -22,7 +22,7 @@ import ConfigParser import logging try: - from leap.mx import couchdb + from leap.mx import couchdbhelper from leap.mx.alias_resolver import AliasResolverFactory except ImportError, ie: print "%s \nExiting... \n" % ie.message @@ -35,8 +35,6 @@ except ImportError, ie: print "This software requires Twisted>=12.0.2, please see the README for" print "help on using virtualenv and pip to obtain requirements." -logger = logging.getLogger(__name__) - if __name__ == "__main__": epilog = "Copyright 2012 The LEAP Encryption Access Project" @@ -54,6 +52,8 @@ if __name__ == "__main__": opts, _ = parser.parse_known_args() + logger = logging.getLogger(name='leap') + debug = opts.debug config_file = opts.config @@ -92,11 +92,11 @@ if __name__ == "__main__": server = config.get("couchdb", "server") port = config.get("couchdb", "port") - cdb = couchdb.ConnectedCouchDB(server, - port=port, - dbName="users", - username=users_user, - password=users_password) + cdb = couchdbhelper.ConnectedCouchDB(server, + port=port, + dbName="users", + username=users_user, + password=users_password) # TODO: use the couchdb for mail -- cgit v1.2.3 From b1c2e24fb08f69a10a4d91a84ec0e31d91991190 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Wed, 24 Apr 2013 16:47:07 -0300 Subject: Improve the rest and start everything from start_mx --- src/leap/mx/__init__.py | 4 +- src/leap/mx/couchdbhelper.py | 14 +-- src/leap/mx/exceptions.py | 23 ----- src/leap/mx/mail_receiver.py | 57 +++++++---- src/leap/mx/runner.py | 83 ---------------- src/leap/mx/util/config.py | 221 ------------------------------------------- src/leap/mx/util/log.py | 143 ---------------------------- src/leap/mx/util/net.py | 126 ------------------------ src/leap/mx/util/storage.py | 42 -------- start_mx.py | 42 +++++--- 10 files changed, 81 insertions(+), 674 deletions(-) delete mode 100644 src/leap/mx/exceptions.py delete mode 100644 src/leap/mx/runner.py delete mode 100644 src/leap/mx/util/config.py delete mode 100644 src/leap/mx/util/log.py delete mode 100644 src/leap/mx/util/net.py delete mode 100644 src/leap/mx/util/storage.py diff --git a/src/leap/mx/__init__.py b/src/leap/mx/__init__.py index 8ef1bc9..0590c90 100644 --- a/src/leap/mx/__init__.py +++ b/src/leap/mx/__init__.py @@ -6,6 +6,4 @@ Module initialization file for leap.mx . """ from leap.mx.util import version -__all__ = ['alias_resolver', 'couchdb', 'exceptions', 'runner', 'util'] -__author__ = version.getAuthors() -__version__ = version.getVersion() +__all__ = ['alias_resolver', 'couchdbhelper'] diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index 4e76be3..2bbca77 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -20,6 +20,10 @@ Classes for working with CouchDB or BigCouch instances which store email alias maps, user UUIDs, and GPG keyIDs. """ +import logging + +from functools import partial + try: from paisley import client except ImportError: @@ -32,9 +36,7 @@ except ImportError: print "This software requires Twisted. Please see the README file" print "for instructions on getting required dependencies." -from functools import partial - -from leap.mx.util import log +logger = logging.getLogger(__name__) class ConnectedCouchDB(client.CouchDB): @@ -82,9 +84,9 @@ class ConnectedCouchDB(client.CouchDB): @param data: response from the listDB command @type data: array """ - log.msg("Available databases:") + logger.msg("Available databases:") for database in data: - log.msg(" * %s" % (database,)) + logger.msg(" * %s" % (database,)) def createDB(self, dbName): """ @@ -117,7 +119,7 @@ class ConnectedCouchDB(client.CouchDB): reduce=False, include_docs=True) - d.addCallbacks(partial(self._get_uuid, alias), log.err) + d.addCallbacks(partial(self._get_uuid, alias), logger.error) return d diff --git a/src/leap/mx/exceptions.py b/src/leap/mx/exceptions.py deleted file mode 100644 index 63b946c..0000000 --- a/src/leap/mx/exceptions.py +++ /dev/null @@ -1,23 +0,0 @@ -#! -*- encoding: utf-8 -*- -""" -Custom exceptions for leap_mx. - -@authors: Isis Lovecruft, 0x2cdb8b35 -@version: 0.0.1 -@license: see included LICENSE file -""" - - -class MissingConfig(Exception): - """Raised when the config file cannot be found.""" - def __init__(self, message=None, config_file=None): - if message: - return - else: - self.message = "Cannot locate config file" - if config_file: - self.message += " %s" % config_file - self.message += "." - -class UnsupportedOS(Exception): - """Raised when we're not *nix or *BSD.""" diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 09200ac..ae32f25 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -17,10 +17,13 @@ # along with this program. If not, see . import os +import uuid as pyuuid import logging import argparse import ConfigParser +import json + from email import message_from_string from functools import partial @@ -28,41 +31,58 @@ from twisted.internet import inotify, reactor from twisted.python import filepath from leap.mx import couchdbhelper + +from leap.soledad import LeapDocument +from leap.soledad.backends.leap_backend import EncryptionSchemes from leap.soledad.backends.couch import CouchDatabase +from leap.common.keymanager import openpgp logger = logging.getLogger(__name__) -def _get_pubkey(uuid): - # TODO: implent! +def _get_pubkey(uuid, cdb): logger.debug("Fetching pubkey for %s" % (uuid,)) - return uuid, "" + return uuid, cdb.getPubKey(uuid) -def _encrypt_message(uuid_pubkey, message): - # TODO: implement! +def _encrypt_message(uuid_pubkey, address_message): uuid, pubkey = uuid_pubkey + address, message = address_message logger.debug("Encrypting message to %s's pubkey" % (uuid,)) logger.debug("Pubkey: %s" % (pubkey,)) if pubkey is None or len(pubkey) == 0: - # TODO: This is only for testing!! REMOVE! - return uuid, message + logger.exception("No public key found") + raise Exception("No public key found") + + doc = LeapDocument(encryption_scheme=EncryptionSchemes.PUBKEY, + doc_id=str(pyuuid.uuid4())) + + def _ascii_to_openpgp_cb(gpg): + key = gpg.list_keys().pop() + return openpgp._build_key_from_gpg(address, key, pubkey) - encrypted = "" + openpgp_key = openpgp._safe_call(_ascii_to_openpgp_cb, pubkey) - return uuid, encrypted + data = {'incoming': True, 'content': message} + doc.content = { + "_encrypted_json": openpgp.encrypt_asym(json.dumps(data), openpgp_key) + } -def _export_message(uuid_message, couch_url): - uuid, message = uuid_message + return uuid, doc + + +def _export_message(uuid_doc, couch_url): + uuid, doc = uuid_doc logger.debug("Exporting message for %s" % (uuid,)) if uuid is None: uuid = 0 - db_url = couch_url + '/user-%s' % uuid - db = CouchDatabase.open_database(db_url, create=True) - doc = db.create_doc({'content': str(message)}) + db = CouchDatabase(couch_url, "user-%s" % (uuid,)) + db.put_doc(doc) + + logger.debug("Done exporting") return True @@ -73,6 +93,7 @@ def _conditional_remove(do_remove, filepath): try: logger.debug("Removing %s" % (filepath.path,)) filepath.remove() + logger.debug("Done removing") except Exception as e: # TODO: better handle exceptions logger.exception("%s" % (e,)) @@ -85,10 +106,14 @@ def _process_incoming_email(users_db, mail_couchdb_url_prefix, self, filepath, m mail_data = f.read() mail = message_from_string(mail_data) owner = mail["Delivered-To"] + owner = owner.split("@")[0] + owner = owner.split("+")[0] + logger.debug("Mail owner: %s" % (owner,)) + logger.debug("%s received a new mail" % (owner,)) d = users_db.queryByLoginOrAlias(owner) - d.addCallback(_get_pubkey) - d.addCallback(_encrypt_message, (mail_data)) + d.addCallback(_get_pubkey, (users_db)) + d.addCallback(_encrypt_message, (owner, mail_data)) d.addCallback(_export_message, (mail_couchdb_url_prefix)) d.addCallback(_conditional_remove, (filepath)) diff --git a/src/leap/mx/runner.py b/src/leap/mx/runner.py deleted file mode 100644 index daf956e..0000000 --- a/src/leap/mx/runner.py +++ /dev/null @@ -1,83 +0,0 @@ -#-*- coding: utf-8 -*- -""" -runner ------- -A module containing application and daemon process utilities. - -@author Isis Agora Lovecruft , 0x2cdb8b35 -@version 0.0.1 - -""" - -from os import path as ospath - -import re - - -class CheckRequirements(ImportError): - """ - Raised when we're missing something from requirements.pip. - """ - def __init__(self, package_name, pipfile, message=None): - """ - Display an error message with instructions for obtaining missing - dependencies. - - @param message: A string describing the error. - @param missing: A string indicating which dependency is missing. - @param pipfile: The path and filename of the pip requirements file, - relative to the top-level repository directory. - """ - if message: - self.message = message - return self - - self.package_name = package_name - self.pipfile = pipfile - self.dependencies = self.__read_pip_requirements__() - self.missing = [] - - for package, version in self.dependencies: - pkg = package.lower() if package == "Twisted" else package - try: - __import__(pkg) - except ImportError: - self.missing.append(package) - - if len(self.missing) > 0: - self.message = self.package_name + " requires " - elif len(self.missing) <= 0: - return None - - if len(self.missing) >= 1: - for missed in self.missing[:-1]: - self.message += missed + ", " - self.message += "and " - - if len(self.missing) == 1: - self.message += self.missing[0] + "." - self.message += "\nPlease see %s for ".format(self.pipfile) - self.message += "instruction on installing dependencies." - raise self(self.message) - - def __read_pip_requirements__(self, file=None): - """ - Check the pip requirements file to determine our dependencies. - - @param file: The full path of the pip requirements.txt file. - @returns: A list of tuple(package_name, package_version). - """ - if not file: - file = self.pipfile - - requirement = re.compile('[^0-9=><]+') - dependencies = [] - - with open(file) as pipfile: - for line in pipfile.readlines(): - shortened = line.strip() - matched = requirement.match(shortened) - package_name = matched.group() - package_version = shortened.split(package_name, 1)[1] - dependencies.append((package_name, package_version)) - return dependencies diff --git a/src/leap/mx/util/config.py b/src/leap/mx/util/config.py deleted file mode 100644 index f655ca9..0000000 --- a/src/leap/mx/util/config.py +++ /dev/null @@ -1,221 +0,0 @@ -#! -*- encoding: utf-8 -*- -""" -Config file utilities. - -This module has an :attr:`config_filename`, which can be used to set the -filename outside of function calls: - - >>> from leap.mx.util import config - >>> config.config_filename = "blahblah.yaml" - -If not set anywhere, it will default to using the top level repository -directory, i.e. "/.../leap_mx/leap_mx.conf", and will create that file with -the default settings if it does not exist. - -The config file can be loaded/created with :func:`config.loadConfig`: - - >>> config.loadConfig() - -Once the config file is loaded, this module presents a highly object-oriented -interface, so that sections taken from the config file become attribute of -this module, and the name of their respective settings become attributes of -the section names. Like this: - - >>> print config.basic.postfix_port - 465 - -@authors: Isis Lovecruft, 0x2cdb8b35 -@version: 0.0.1 -@license: see included LICENSE file -""" - -from os import path as ospath - -import sys -import yaml - -from leap.mx.util import version, storage -from leap.mx.exceptions import MissingConfig, UnsupportedOS - - -filename = None -config_version = None -basic = storage.Storage() -couch = storage.Storage() -advanced = storage.Storage() - -PLATFORMS = {'LINUX': sys.platform.startswith("linux"), - 'OPENBSD': sys.platform.startswith("openbsd"), - 'FREEBSD': sys.platform.startswith("freebsd"), - 'NETBSD': sys.platform.startswith("netbsd"), - 'DARWIN': sys.platform.startswith("darwin"), - 'SOLARIS': sys.platform.startswith("sunos"), - 'WINDOWS': sys.platform.startswith("win32")} - -def getClientPlatform(platform_name=None): - """ - Determine the client's operating system platform. Optionally, if - :param:`platform_name` is given, check that this is indeed the platform - we're operating on. - - @param platform_name: A string, upper-, lower-, or mixed case, of one - of the keys in the :attr:`leap.util.version.PLATFORMS` - dictionary. E.g. 'Linux' or 'OPENBSD', etc. - @returns: A string specifying the platform name, and the boolean test - used to determine it. - """ - for name, test in PLATFORMS.items(): - if not platform_name or platform_name.upper() == name: - if test: - return name, test - -def _create_config_file(conffile): - """ - Create the config file if it doesn't exist. - - @param conffile: The full path to the config file to write to. - """ - with open(conffile, 'w+') as conf: - conf.write(""" -# -# mx.conf -# ======= -# Configurable options for the leap_mx encrypting mail exchange. -# -# This file follows YAML markup format: http://yaml.org/spec/1.2/spec.html -# Keep in mind that indentation matters. -# - -basic: - # Whether or not to log to file: - enable_logfile: True - # The name of the logfile: - logfile: mx.log - # Where is the spoolfile of messages to encrypt?: - spoolfile: /var/mail/encrypt_me -couch: - # The couch username for authentication to a CouchDB instance: - user: admin - # The couch username's password: - passwd: passwd - # The CouchDB hostname or IP address to connect to: - host: couchdb.example.com - # The CouchDB port to connect to: - port: 7001 -advanced: - # Which port on localhost should postfix send check_recipient queries to?: - check_recipient_access_port: 1347 - # Which port on localhost should postfix ask for UUIDs?: - virtual_alias_map_port: 1348 - # Enable debugging output in the logger: - debug: True - # Print enough things really fast to make you look super 1337: - noisy: False -config_version: 0.0.2 - -""") - conf.flush() - assert ospath.isfile(conffile), "Config file %s not created!" % conffile - -def _get_config_location(config_filename=None, - use_dot_config_directory=False): - """ - Get the full path and filename of the config file. - """ - platform = getClientPlatform()[0] - - ## If not given, default to the application's name + '.conf' - if not config_filename: - if not filename: - config_filename = "mx.conf" - else: - config_filename = filename - - ## Oh hell, it could be said only to beguile: - ## That windoze users are capable of editing a .conf file. - ## Also, what maddened wingnut would be so fool - ## To run a mail exchange on a windoze nodule? - ## I'm ignoring these loons for now. And pardon if I seem jaded, - ## But srsly, this and that solaris sh*t should be deprecated. - if not platform.endswith('LINUX') and not platform.endswith('BSD'): - raise UnsupportedOS("Sorry, your operating system isn't supported.") - - where = None - if use_dot_config_directory: - ## xxx only install/import this in *nix - from xdg import BaseDirectory - - dot_config_dirs = BaseDirectory.xdg_config_dirs - for dir in dot_config_dirs: - our_dir = ospath.join(dir, package_name) - if ospath.isdir(our_dir): - if config_filename in os.listdir(our_dir): - where = ospath.abspath(our_dir) - ## Use repo dir instead: - if not where: - where = version.getRepoDir() - - conffile = ospath.join(where, config_filename) - try: - with open(conffile) as cf: pass - except IOError: - _create_config_file(conffile) - finally: - return conffile - -def loadConfig(file=None): - """ - Some of this is taken from OONI config code for now, and so this should be - refacotored, along with the leap_client config code, so that we have - similarly structured config files. It is perhaps desirable to also use - soledad as a backend for remote setup and maintainance, and thus this code - will need to hook into u1db (and potentially "pysqlcipher"). - - Excuse the yaml for now, I just wanted something that works. - - @param file: (optional) If provided, use this filename. - """ - if not file: - file = _get_config_location() - - if ospath.isfile(file): - with open(file, 'a+') as conf: - config_contents = '\n'.join(conf.readlines()) - cfg = yaml.safe_load(config_contents) - - ## These become objects with their keys loaded as attributes: - ## - ## from leap.util import config - ## config.basic.foo = bar - ## - try: - for k, v in cfg['basic'].items(): - basic[k] = v - except (AttributeError, KeyError): pass - - try: - for k, v in cfg['advanced'].items(): - advanced[k] = v - except (AttributeError, KeyError): pass - - try: - for k, v in cfg['couch'].items(): - couch[k] = v - except (AttributeError, KeyError): pass - - if 'config_version' in cfg: - config_version = cfg['config_version'] - else: - config_version = 'unknown' - - return basic, couch, advanced, config_version - else: - raise MissingConfig("Could not load config file.") - - -## This is the name of the config file to use: -## If not set, it defaults to 'leap_mx/leap_mx.conf' -if not filename: - filename = _get_config_location() -else: - filename = _get_config_location(config_filename=filename) diff --git a/src/leap/mx/util/log.py b/src/leap/mx/util/log.py deleted file mode 100644 index f31684d..0000000 --- a/src/leap/mx/util/log.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- encoding: utf-8 -*- -''' -log.py ------- -Logging for leap_mx. - -@authors: Isis Agora Lovecruft, 0x2cdb8b35 -@licence: see included LICENSE file -@copyright: 2013 Isis Agora Lovecruft -''' - -from datetime import datetime -from functools import wraps - -import logging -import os -import sys -import time -import traceback - -from twisted.python import log as txlog -from twisted.python import util as txutil -from twisted.python import logfile as txlogfile -from twisted.python.failure import Failure - -from leap.mx.util import version, config - - -class InvalidTimestampFormat(Exception): - pass - -class UnprefixedLogfile(txlog.FileLogObserver): - """Logfile with plain messages, without timestamp prefixes.""" - def emit(self, eventDict): - text = txlog.textFromEventDict(eventDict) - if text is None: - return - - txutil.untilConcludes(self.write, "%s\n" % text) - txutil.untilConcludes(self.flush) - - -def utcDateNow(): - """The current date for UTC time.""" - return datetime.utcnow() - -def utcTimeNow(): - """Seconds since epoch in UTC time, as type float.""" - return time.mktime(time.gmtime()) - -def dateToTime(date): - """Convert datetime to seconds since epoch.""" - return time.mktime(date.timetuple()) - -def prettyDateNow(): - """Pretty string for the local time.""" - return datetime.now().ctime() - -def utcPrettyDateNow(): - """Pretty string for UTC.""" - return datetime.utcnow().ctime() - -def timeToPrettyDate(time_val): - """Convert seconds since epoch to date.""" - return time.ctime(time_val) - -def start(logfilename=None, logfiledir=None): - """ - Start logging to stdout, and optionally to a logfile as well. - - @param logfile: The full path of the filename to store logs in. - """ - txlog.startLoggingWithObserver(UnprefixedLogfile(sys.stdout).emit) - - if logfilename and logfiledir: - if not os.path.isdir(logfiledir): - os.makedirs(logfiledir) - daily_logfile = txlogfile.DailyLogFile(logfilename, logfiledir) - txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit) - - txlog.msg("Starting %s, version %s, on %s UTC" % (version.getPackageName(), - version.getVersion(), - utcPrettyDateNow())) - txlog.msg("Authors: %s" % version.getAuthors()) - -def msg(msg, *arg, **kwarg): - """Log a message at the INFO level.""" - print "[*] %s" % msg - -def debug(msg, *arg, **kwarg): - """Log a message at the DEBUG level.""" - if config.advanced.debug: - print "[d] %s" % msg - -def warn(msg, *arg, **kwarg): - """Log a message at the WARN level.""" - if config.basic.show_warnings: - txlog.logging.captureWarnings('true') - print "[#] %s" % msg - -def err(msg, *arg, **kwarg): - """Log a message at the ERROR level.""" - print "[!] %s" % msg - -def fail(*failure): - """Log a message at the CRITICAL level.""" - logging.critical(failure) - ## xxx should we take steps to exit here? - -def exception(error): - """ - Catch an exception and print only the error message, then continue normal - program execution. - - @param error: Can be error messages printed to stdout and to the - logfile, or can be a twisted.python.failure.Failure instance. - """ - if isinstance(error, Failure): - error.printTraceback() - else: - exc_type, exc_value, exc_traceback = sys.exc_info() - traceback.print_exception(exc_type, exc_value, exc_traceback) - -def catch(func): - """ - Quick wrapper to add around test methods for debugging purposes, - catches the given Exception. Use like so: - - >>> @log.catch - def foo(bar): - if bar == 'baz': - raise Exception("catch me no matter what I am") - >>> foo("baz") - [!] catch me no matter what I am - - """ - @wraps(func) - def _catch(*args, **kwargs): - try: - func(*args, **kwargs) - except Exception, exc: - exception(exc) - return _catch diff --git a/src/leap/mx/util/net.py b/src/leap/mx/util/net.py deleted file mode 100644 index 64dbc90..0000000 --- a/src/leap/mx/util/net.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -''' -net.py -------- -Utilities for networking. - -@authors: Isis Agora Lovecruft, 0x2cdb8b35 -@license: see included LICENSE file -@copyright: 2013 Isis Agora Lovecruft -''' - -import ipaddr -import sys -import socket - -from random import randint - -from leap.mx.util import log - - -PLATFORMS = {'LINUX': sys.platform.startswith("linux"), - 'OPENBSD': sys.platform.startswith("openbsd"), - 'FREEBSD': sys.platform.startswith("freebsd"), - 'NETBSD': sys.platform.startswith("netbsd"), - 'DARWIN': sys.platform.startswith("darwin"), - 'SOLARIS': sys.platform.startswith("sunos"), - 'WINDOWS': sys.platform.startswith("win32")} - - -class UnsupportedPlatform(Exception): - """Support for this platform is not currently available.""" - -class IfaceError(Exception): - """Could not find default network interface.""" - -class PermissionsError(SystemExit): - """This test requires admin or root privileges to run. Exiting...""" - - -def checkIPaddress(addr): - """ - Check that a given string is a valid IPv4 or IPv6 address. - - @param addr: Any string defining an IP address, i.e. '1.2.3.4' or '::1'. - @returns: True if :param:`addr` defines a valid IPAddress, else False. - """ - import ipaddr - - try: - check = ipaddr.IPAddress(addr) - except ValueError, ve: - log.warn(ve.message) - return False - else: - return True - -def getClientPlatform(platform_name=None): - for name, test in PLATFORMS.items(): - if not platform_name or platform_name.upper() == name: - if test: - return name, test - -def getPosixIfaces(): - from twisted.internet.test import _posixifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _posixifaces._interfaces() - return ifaces - -def getWindowsIfaces(): - from twisted.internet.test import _win32ifaces - log.msg("Attempting to discover network interfaces...") - ifaces = _win32ifaces._interfaces() - return ifaces - -def getIfaces(platform_name=None): - client, test = getClientPlatform(platform_name) - if client: - if client == ('LINUX' or 'DARWIN') or client[-3:] == 'BSD': - return getPosixIfaces() - elif client == 'WINDOWS': - return getWindowsIfaces() - ## XXX fixme figure out how to get iface for Solaris - else: - raise UnsupportedPlatform - else: - raise UnsupportedPlatform - -def getRandomUnusedPort(addr=None): - free = False - while not free: - port = randint(1024, 65535) - s = socket.socket() - try: - s.bind((addr, port)) - free = True - except: - pass - s.close() - return port - -def getNonLoopbackIfaces(platform_name=None): - try: - ifaces = getIfaces(platform_name) - except UnsupportedPlatform, up: - log.err(up) - - if not ifaces: - log.msg("Unable to discover network interfaces...") - return None - else: - found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] - log.debug("Found non-loopback interfaces: %s" % found) - for iface in ifaces: - try: - interface = checkInterfaces(found) - except IfaceError, ie: - log.err(ie) - return None - else: - return interfaces - - -def getLocalAddress(): - default_iface = getDefaultIface() - return default_iface.ipaddr diff --git a/src/leap/mx/util/storage.py b/src/leap/mx/util/storage.py deleted file mode 100644 index c4c797a..0000000 --- a/src/leap/mx/util/storage.py +++ /dev/null @@ -1,42 +0,0 @@ - -class Storage(dict): - """ - A Storage object is like a dictionary except `obj.foo` can be used - in addition to `obj['foo']`. - - >>> o = Storage(a=1) - >>> o.a - 1 - >>> o['a'] - 1 - >>> o.a = 2 - >>> o['a'] - 2 - >>> del o.a - >>> o.a - None - """ - def __getattr__(self, key): - try: - return self[key] - except KeyError, k: - return None - - def __setattr__(self, key, value): - self[key] = value - - def __delattr__(self, key): - try: - del self[key] - except KeyError, k: - raise AttributeError, k - - def __repr__(self): - return '' - - def __getstate__(self): - return dict(self) - - def __setstate__(self, value): - for (k, v) in value.items(): - self[k] = v diff --git a/start_mx.py b/start_mx.py index c802852..067011f 100755 --- a/start_mx.py +++ b/start_mx.py @@ -21,16 +21,20 @@ import sys import ConfigParser import logging +from functools import partial + try: - from leap.mx import couchdbhelper + from leap.mx import couchdbhelper, mail_receiver from leap.mx.alias_resolver import AliasResolverFactory except ImportError, ie: print "%s \nExiting... \n" % ie.message + raise sys.exit(1) try: - from twisted.internet import reactor + from twisted.internet import reactor, inotify from twisted.internet.endpoints import TCP4ServerEndpoint + from twisted.python import filepath except ImportError, ie: print "This software requires Twisted>=12.0.2, please see the README for" print "help on using virtualenv and pip to obtain requirements." @@ -83,11 +87,8 @@ if __name__ == "__main__": config = ConfigParser.ConfigParser() config.read(config_file) - users_user = config.get("couchdb", "users_user") - users_password = config.get("couchdb", "users_password") - - mail_user = config.get("couchdb", "mail_user") - mail_password = config.get("couchdb", "mail_password") + user = config.get("couchdb", "user") + password = config.get("couchdb", "password") server = config.get("couchdb", "server") port = config.get("couchdb", "port") @@ -95,12 +96,31 @@ if __name__ == "__main__": cdb = couchdbhelper.ConnectedCouchDB(server, port=port, dbName="users", - username=users_user, - password=users_password) + username=user, + password=password) + + # Mail receiver + wm = inotify.INotify(reactor) + wm.startReading() + + mask = inotify.IN_CREATE + + mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user, + password, + server, + port) + + incoming_partial = partial(mail_receiver._process_incoming_email, cdb, mail_couch_url_prefix) + for section in config.sections(): + if section in ("couchdb"): + continue + to_watch = config.get(section, "path") + recursive = config.getboolean(section, "recursive") + logger.debug("Watching %s --- Recursive: %s" % (to_watch, recursive)) + wm.watch(filepath.FilePath(to_watch), mask, callbacks=[incoming_partial], recursive=recursive) - # TODO: use the couchdb for mail - # TODO: make the listening ports configurable + # Alias map alias_endpoint = TCP4ServerEndpoint(reactor, 4242) alias_endpoint.listen(AliasResolverFactory(couchdb=cdb)) -- cgit v1.2.3 From be45dc148a3586149f58944788f3e9bceea4f410 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 25 Apr 2013 11:22:21 -0300 Subject: Catch the case where there's no pubkey for a user --- src/leap/mx/couchdbhelper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index 2bbca77..2f6b548 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -138,7 +138,10 @@ class ConnectedCouchDB(client.CouchDB): for row in result["rows"]: if row["key"] == alias: uuid = row["id"] - self._cache[uuid] = row["doc"]["public_key"] + try: + self._cache[uuid] = row["doc"]["public_key"] + except: + pass # no public key for this user return uuid return None -- cgit v1.2.3 From 833f5a3cb8352cf81962817ec7ed69b982891a80 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 25 Apr 2013 11:35:07 -0300 Subject: Add check recipient access --- src/leap/mx/check_recipient_access.py | 59 +++++++++++++++++++++++++++++++++++ start_mx.py | 5 +++ 2 files changed, 64 insertions(+) create mode 100644 src/leap/mx/check_recipient_access.py diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py new file mode 100644 index 0000000..8d03297 --- /dev/null +++ b/src/leap/mx/check_recipient_access.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# check_recipient_access.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Classes for resolving postfix recipient access +""" + +import logging + +try: + from twisted.protocols import postfix +except ImportError: + print "This software requires Twisted. Please see the README file" + print "for instructions on getting required dependencies." + +logger = logging.getLogger(__name__) + + +class CheckRecipientAccess(postfix.PostfixTCPMapServer): + def _cbGot(self, value): + if value is None: + self.sendCode(500) + else: + self.sendCode(200) + + +class CheckRecipientAccessFactory(postfix.PostfixTCPMapDeferringDictServerFactory): + + protocol = CheckRecipientAccess + + def __init__(self, couchdb, *args, **kwargs): + postfix.PostfixTCPMapDeferringDictServerFactory.__init__(self, *args, **kwargs) + self._cdb = couchdb + + def get(self, key): + orig_key = key + try: + key = key.split("@")[0] + key = key.split("+")[0] + except Exception as e: + key = orig_key + logger.exception("%s" % (e,)) + d = self._cdb.queryByLoginOrAlias(key) + return d diff --git a/start_mx.py b/start_mx.py index 067011f..20ea554 100755 --- a/start_mx.py +++ b/start_mx.py @@ -26,6 +26,7 @@ from functools import partial try: from leap.mx import couchdbhelper, mail_receiver from leap.mx.alias_resolver import AliasResolverFactory + from leap.mx.check_recipient_access import CheckRecipientAccessFactory except ImportError, ie: print "%s \nExiting... \n" % ie.message raise @@ -124,4 +125,8 @@ if __name__ == "__main__": alias_endpoint = TCP4ServerEndpoint(reactor, 4242) alias_endpoint.listen(AliasResolverFactory(couchdb=cdb)) + # Check recipient access + check_recipient = TCP4ServerEndpoint(reactor, 2244) + check_recipient.listen(CheckRecipientAccessFactory(couchdb=cdb)) + reactor.run() -- cgit v1.2.3 From e7756fef684fce2b09494222edff72bf06447aa2 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 25 Apr 2013 12:04:55 -0300 Subject: More cleanup --- src/leap/__init__.py | 28 ++++++++++--- src/leap/mx/__init__.py | 20 +++++++-- src/leap/mx/tests/__init__.py | 24 +++++++---- src/leap/mx/util/__init__.py | 12 ------ src/leap/mx/util/version.py | 95 ------------------------------------------- start_mx.py | 12 ++---- 6 files changed, 59 insertions(+), 132 deletions(-) delete mode 100644 src/leap/mx/util/__init__.py delete mode 100644 src/leap/mx/util/version.py diff --git a/src/leap/__init__.py b/src/leap/__init__.py index f48ad10..ff2d8a1 100644 --- a/src/leap/__init__.py +++ b/src/leap/__init__.py @@ -1,6 +1,22 @@ -# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages -try: - __import__('pkg_resources').declare_namespace(__name__) -except ImportError: - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) +# -*- encoding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Module intialization file for leap. +""" + +__all__ = ['mx'] diff --git a/src/leap/mx/__init__.py b/src/leap/mx/__init__.py index 0590c90..efb28ae 100644 --- a/src/leap/mx/__init__.py +++ b/src/leap/mx/__init__.py @@ -1,7 +1,21 @@ -#-*- encoding: utf-8 -*- +# -*- encoding: utf-8 -*- +# __init__.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + """ -leap/mx/__init__.py -------------------- Module initialization file for leap.mx . """ from leap.mx.util import version diff --git a/src/leap/mx/tests/__init__.py b/src/leap/mx/tests/__init__.py index 0416769..2002c48 100644 --- a/src/leap/mx/tests/__init__.py +++ b/src/leap/mx/tests/__init__.py @@ -1,13 +1,23 @@ -#-*- encoding: utf-8 -*- +# -*- encoding: utf-8 -*- +# start_mx.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + """ -leap/mx/tests/__init__.py -------------------------- Module intialization file for leap.mx.tests, a module containing unittesting code, using twisted.trial, for testing leap_mx. - -@authors: Isis Agora Lovecruft, 0x2cdb8b35 -@license: AGPLv3, see included LICENSE file -@copyright: © 2013 Isis Lovecruft, see COPYLEFT file """ __all__ = ['test_alias_resolver'] diff --git a/src/leap/mx/util/__init__.py b/src/leap/mx/util/__init__.py deleted file mode 100644 index c4a93b8..0000000 --- a/src/leap/mx/util/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -#-*- encoding: utf-8 -*- -""" -leap/mx/util/__init__.py ------------------------- -Module intialization file for leap.mx.util. -""" - -import version -version = version.Version() - -__all__ = ['config', 'log', 'net', 'storage', 'version'] - diff --git a/src/leap/mx/util/version.py b/src/leap/mx/util/version.py deleted file mode 100644 index c32166f..0000000 --- a/src/leap/mx/util/version.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -''' -version.py ----------- -Version information for leap_mx. - -@authors: Isis Agora Lovecruft, 0x2cdb8b35 -@licence: see included LICENSE file -@copyright: 2013 Isis Agora Lovecruft -''' - -from os import getcwd -from os import path as ospath - -import sys - - -class Version(object): - def __init__(self): - self.name = 'leap_mx' - self.version = '0.0.2' - self.pipfile = ospath.join(self.getRepoDir(), - 'pkg/mx-requirements.pip') - self.authors = [ - ('Isis Agora Lovecruft', '', '0x2cdb8b35'), - ] - self.git_url = 'https://github.com/isislovecruft/leap_mx/' - self.website = 'https://leap.se' - - def getPackageName(self): - """Returns the application name.""" - return self.name - - def getPipfile(self): - """Returns the full path of the pip requirements.txt file.""" - return self.pipfile - - def getVersion(self): - """Returns a version the application name and version number.""" - return self.version - - def getAuthors(self): - credits = str() - for author in self.authors: - credits += " ".join(author) - return credits - - def getRepoDir(self): - """Get the top-level repository directory.""" - here = getcwd() - base = here.rsplit(self.name, 1)[0] - repo = ospath.join(base, self.name) - return repo - - def __make_text__(self, extra_text=None): - splitr = "-" * len(self.version.__str__()) - header = ["\n%s\n" % self.version.__str__(), - "%s\n" % splitr] - footer = ["Website: \t%s\n" % self.website, - "Github: \t%s\n" % self.git_url, - "\n"] - contacts = ["\t%s, %s %s\n" - % (a[0], a[1], a[2]) for a in self.authors] - contacts.insert(0, "Authors: ") - - with_contacts = header + contacts - - if extra_text is not None: - if isinstance(extra_text, iter): - with_contacts.extend((e for e in extra_text)) - elif isinstance(extra_text, str): - with_contacts.append(extra_text) - else: - print "Couldn't add extra text..." - - text = with_contacts + footer - return text - - def __update_version__(self): - repo = self.getRepoDir() - self.version_file = ospath.join(repo, 'VERSION') - version_text = self.__make_text__() - - with open(self.version_file, 'w+') as fh: - fh.writelines((line for line in version_text)) - fh.flush() - fh.truncate() - - -if __name__ == "__main__": - print "Generating new VERSION file..." - vrsn = Version() - vrsn.__update_version__() - print "Done." diff --git a/start_mx.py b/start_mx.py index 20ea554..7e154da 100755 --- a/start_mx.py +++ b/start_mx.py @@ -17,20 +17,14 @@ # along with this program. If not, see . import argparse -import sys import ConfigParser import logging from functools import partial -try: - from leap.mx import couchdbhelper, mail_receiver - from leap.mx.alias_resolver import AliasResolverFactory - from leap.mx.check_recipient_access import CheckRecipientAccessFactory -except ImportError, ie: - print "%s \nExiting... \n" % ie.message - raise - sys.exit(1) +from leap.mx import couchdbhelper, mail_receiver +from leap.mx.alias_resolver import AliasResolverFactory +from leap.mx.check_recipient_access import CheckRecipientAccessFactory try: from twisted.internet import reactor, inotify -- cgit v1.2.3 From 7d2756dd120800899f30b74ca68787e1044bed7c Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 25 Apr 2013 12:12:02 -0300 Subject: Reorder files, normalize repo and add sample config --- COPYLEFT | 13 -- DESIGN.md | 238 --------------------- NOTES.md | 59 ----- bootstrap | 59 ----- doc/DESIGN.md | 238 +++++++++++++++++++++ doc/NOTES.md | 59 +++++ gpg-key-generator | 209 ------------------ gpg-keys/generated-2013-02-15_19-15-001.pub | Bin 2779 -> 0 bytes gpg-keys/generated-2013-02-15_19-15-001.sec | Bin 5453 -> 0 bytes gpg-keys/generated-2013-02-15_19-15-002.pub | Bin 2779 -> 0 bytes gpg-keys/generated-2013-02-15_19-15-002.sec | Bin 5453 -> 0 bytes gpg-keys/generated-2013-02-15_19-15-003.pub | Bin 2779 -> 0 bytes gpg-keys/generated-2013-02-15_19-15-003.sec | Bin 5453 -> 0 bytes gpg-keys/gpg-batch-key-script | 45 ---- pkg/utils/bootstrap | 59 +++++ pkg/utils/gpg-key-generator | 209 ++++++++++++++++++ .../gpg-keys/generated-2013-02-15_19-15-001.pub | Bin 0 -> 2779 bytes .../gpg-keys/generated-2013-02-15_19-15-001.sec | Bin 0 -> 5453 bytes .../gpg-keys/generated-2013-02-15_19-15-002.pub | Bin 0 -> 2779 bytes .../gpg-keys/generated-2013-02-15_19-15-002.sec | Bin 0 -> 5453 bytes .../gpg-keys/generated-2013-02-15_19-15-003.pub | Bin 0 -> 2779 bytes .../gpg-keys/generated-2013-02-15_19-15-003.sec | Bin 0 -> 5453 bytes pkg/utils/gpg-keys/gpg-batch-key-script | 45 ++++ pkg/utils/test_bootstrap | 39 ++++ sample-config/mx.conf | 9 + test_bootstrap | 39 ---- 26 files changed, 658 insertions(+), 662 deletions(-) delete mode 100644 COPYLEFT delete mode 100644 DESIGN.md delete mode 100644 NOTES.md delete mode 100755 bootstrap create mode 100644 doc/DESIGN.md create mode 100644 doc/NOTES.md delete mode 100755 gpg-key-generator delete mode 100644 gpg-keys/generated-2013-02-15_19-15-001.pub delete mode 100644 gpg-keys/generated-2013-02-15_19-15-001.sec delete mode 100644 gpg-keys/generated-2013-02-15_19-15-002.pub delete mode 100644 gpg-keys/generated-2013-02-15_19-15-002.sec delete mode 100644 gpg-keys/generated-2013-02-15_19-15-003.pub delete mode 100644 gpg-keys/generated-2013-02-15_19-15-003.sec delete mode 100644 gpg-keys/gpg-batch-key-script create mode 100755 pkg/utils/bootstrap create mode 100755 pkg/utils/gpg-key-generator create mode 100644 pkg/utils/gpg-keys/generated-2013-02-15_19-15-001.pub create mode 100644 pkg/utils/gpg-keys/generated-2013-02-15_19-15-001.sec create mode 100644 pkg/utils/gpg-keys/generated-2013-02-15_19-15-002.pub create mode 100644 pkg/utils/gpg-keys/generated-2013-02-15_19-15-002.sec create mode 100644 pkg/utils/gpg-keys/generated-2013-02-15_19-15-003.pub create mode 100644 pkg/utils/gpg-keys/generated-2013-02-15_19-15-003.sec create mode 100644 pkg/utils/gpg-keys/gpg-batch-key-script create mode 100755 pkg/utils/test_bootstrap create mode 100644 sample-config/mx.conf delete mode 100755 test_bootstrap diff --git a/COPYLEFT b/COPYLEFT deleted file mode 100644 index d8e7331..0000000 --- a/COPYLEFT +++ /dev/null @@ -1,13 +0,0 @@ - - This file is part of leap_mx, an encrypting mail exchange program. - Copyright (C) 2013 Isis Lovecruft - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. diff --git a/DESIGN.md b/DESIGN.md deleted file mode 100644 index 2d9fe82..0000000 --- a/DESIGN.md +++ /dev/null @@ -1,238 +0,0 @@ -# design # - -## overview # ----------------------- -This page pertains to the incoming mail exchange servers of the provider. - -General overview of how incoming email will work: - - 1. Incoming message is received by provider's MX servers. - 2. The MTA (postfix in our case) does a ton of checks on the message before we - even check to see if the recipient is valid (this comes from experience - running the riseup mail infrastructure, where the vast majority of messages - can be rejected early in the SMTP reception and thus save a ton of processing - time on the server). - 3. Postfix then queries the database to check if the recipient is valid, if - they are over quota, if their account is enabled, and to resolve any aliases - for the account. - 4. The message is then delivered to an on-disk message spool. - 5. A daemon watches for new files in this spool. Each message is encrypted to - the user's public key, and stored in the user's incoming message queue (stored - in couchdb), and removed from disk. - 6. When the user next logs in with their client, the user's message queue is - emptied by the client. - 7. Each message is decrypted by the client, and then stored in the user's - "inbox" as an unread message. - 8. This local inbox uses soledad for storage - 9. Soledad, in the background, will then re-encrypt this email (now a soledad - document), and sync to the cloud. - -## postfix pipeline ## ---------------------------- -incoming mx servers will run postfix, configured in a particular way: - - 1. postscreen: before accepting an incoming message, checks RBLs, checks RFC - validity, checks for spam pipelining. - (pass) proceed to next step. - (fail) return SMTP error, which bounces email. - 2. more SMTP checks: valid hostnames, etc. - (pass) accepted, proceed to next step. - (fail) return SMTP error, which bounces email. - 3. check_recipient_access -- look up each recipient and ensure they are - allowed to receive messages. - (pass) empty result, proceed to next step. - (fail) return SMTP error code and error comment, bounce message. - 4. milter processessing (spamassassin & clamav) - (pass) continue - (fail) bounce message, flag as spam, or silently kill. - 5. virtual_alias_maps -- map user defined aliases and forwards - (local address) continue if new address is for this mx - (remote address) continue. normally, postfix would relay to the remote domain, but we don't want that. - 6. deliver message to spool - (write) save the message to disk on the mx. - 7. postfix's job is done, mail_receiver picks up email from spool directory - -Questions: - - * what is the best way to have postfix write a message to a spool directory? - There is a built-in facility for saving to a maildir, so we could just - specify a common maildir for everyone. alternately, we could pipe to a - simple command that was responsible for safely saving the file to disk. a - third possibility would be to have a local long running daemon that spoke - lmtp that postfix forward the message on to for delivery. - * if virtual_alias_maps comes after check_recipient_access, then a user with - aliases set but who is over quota will not be able to forward email. i think - this is fine. - * if we are going to support forwarding, we should ensure that the message - gets encrypted before getting forwarded. so, postfix should not do any - forwarding. instead, this should be the job of mail_receiver. - -Considerations: - - 1. high load should fill queue, not crash pipeline: It is important that the - pipeline be able to handle massive bursts of email, as often happens with - email. This means map lookups need to be very fast, and when there is a high - load of email postfix should not be waiting on the mail receiver but must be - able to pass the message off quickly and have the slower mail receiver churn - through the backlog as best it can. - 2. don't lose messages: It is important to not lose any messages when there is - a problem. So, generally, a copy of an email should always exist in some spool - somewhere, and that copy should not be deleted until there is confirmation - that the next stage has succeeded. - -## alias_resolver ## ------------------------------- -The alias_resolver will be a daemon running on MX servers that handles lookups -in the user database of email aliases, forwards, quota, and account status. - -Communication with: - - 1. postfix:: alias_resolver will be bound to localhost and speak postfix's - very simple [tcp map protocol -> http://www.postfix.org/tcp_table.5.html]. - - 2. couchdb:: alias_resolver will make couchdb queries to a local http load - balancer that connects to a couchdb/bigcouch - cluster. [directly accessing the couch->https://we.riseup.net/leap+platform/querying-the-couchdb] - might help getting started. - -### Discussion: ### - - 1. we want the lookups to be fast. using views in couchdb, these should be - very fast. when using bigcouch, we can make it faster by specifying a read - quorum of 1 (instead of the default 2). this will make it so that only a - single couchdb needs to be queried to find the result. i don't know if this - would cause problems, but aliases don't change very often. - -alias_resolver will be responsible for two map lookups in postfix: - -#### check_recipient #### -------------------------- -postfix config: - -@check_recipient_access tcp:localhost:1000@ - -postfix will send "get username@domain.org" and alias_resolver should return an -empty result ("200 \n", i think) if postfix should deliver email to the -user. otherwise, it should return an error. here is example response, verbatim, -that can be used to bounce over quota users: - -``` -200 DEFER_IF_PERMIT Sorry, your message cannot be delivered because the -recipient's mailbox is full. If you can contact them another way, you may wish -to tell them of this problem. -``` - -"DEFER_IF_PERMIT" will let the other MX know that this error is temporary and -that they should try again soon. Typically, an MX will try repeatedly, at -longer and longer intervals, for four days before giving up. - -#### virtual alias map #### ---------------------------- -postfix config: - -@virtual_alias_map tcp:localhost:1001@ - -postfix will send "get alias-address@domain.org" and alias_resolver should -return "200 id_123456\n", where 123456 is the unique id of the user that has -alias-address@domain.org. - -couchdb should have a view that will let us query on an (alias) address and -return the user id. - -note: if the result of the alias map (e.g. id_123456) does not have a domain -suffix, i think postfix will use the 'default transport'. if we want it to use -the virtual transport instead, we should append the domain (eg -id_123456@example.org). see -http://www.postfix.org/ADDRESS_REWRITING_README.html#resolve - - -### Current status: ### -The current implementation of alias_resolver is in -leap-mx/src/leap/mx/alias_resolver.py. - -The class ```alias_resolver.StatusCodes``` deals with creating SMTP-like -response messages for Postfix, speaking Postfix's TCP Map protocol (from item -#1). - -As for Discussion item #1: - -It might be possible to use -[python-memcached](https://pypi.python.org/pypi/python-memcached/) as an -interface to a [memcached](http://memcached.org/) instance to speed up database -lookups, by keeping an in memory mapping of recent request/response -pairs. Also, Twisted now (I think as of 12.0.0) ships with a protocol for -handling Memcached servers, this is in ```twisted.protocols.memcache```. This -should be prioritised for later, if it is decided that querying the CouchDB is -too expensive or time-consuming. - -Thus far, to speed up alias lookup, an in-memory mapping of alias<->resolution -pairs is created by ```alias_resolver.AliasResolverFactory()```, which can be -optionally seeded with a dictionary of ```{ 'alias': 'resolution' }``` pairs -by doing: -~~~~~~ ->>> from leap.mx import alias_resolver ->>> aliasResolverFactory = alias_resolver.AliasResolverFactory( -... addr='1.2.3.4', port=4242, data={'isis': 'isis@leap.se', -... 'drebs': 'drebs@leap.se'}) ->>> aliasResolver = aliasResolverFactory.buildProtocol() ->>> aliasResolver.check_recipient_access('isis') -200 OK Others might say 'HELLA AWESOME'...but we're not convinced. -~~~~~~ - -TODO: - 1. The AliasResolverFactory needs to be connected to the CouchDB. The - classmethod in which this should occur is ```AliasResolverFactory.get()```. - - 2. I am not sure where to get the user's UUID from (Soledad?). Wherever we get - it from, it will need to be returned in - ```AliasResolver.virtual_alias_map()```, and if we want Postfix to hear about - it, then that response will need to be fed into ```AliasResolver.sendCode```. - - 3. Other than those two things, I think everything is done. The only potential - other thing I can think of is that the codes in - ```alias_resolver.StatusCodes``` might need to be urlencoded for Postfix to - accept them, but this is like two lines of code from urllib. - - - -## mail_receiver ## - -the mail_receiver is a daemon that runs on incoming MX servers and is -responsible for encrypting incoming email to the user's public key and saving -the email to an incoming queue database for that user. - -communicates with: - - * message spool directory:: mail_reciever sits and waits for new email to be - written to the spool directory (maybe using this - https://github.com/seb-m/pyinotify, i think it is better than FAM). when a - new file is dumped into the spool, mail_receiver reads the file, encrypts - the entire thing using the public key of the recipient, and saves to - couchdb. - * couchdb get:: mail_receiver does a query on user id to get back user's - public openpgp key. read quorum of 1 is probably ok. - * couchdb put:: mail_receiver communicates with couchdb for storing encrypted - email for each user (eventually, mail_receiver will communicate with a local - http proxy, that communicates with a bigcouch cluster, but the api is - identical) - -discussion: - * i am not sure if postfix adds a header to indicate to whom a message was - actually delivered. if not, this is a problem, because then how do we know - what db to put it in or what public key to use? this is perhaps a good - reason to not let postfix handle writing the message to disk, but instead - pipe it to another command (because postfix sets env variables for stuff - like recipient). - - * should the incoming message queue be a separate database or should it be - just documents in the user's main database with special flags? - - * whenever possible, we should refer to the user by a fixed id, not their - username, because we want to support the ability to change usernames. so, - for example, database names should not be based on usernames. - -### Current Status: ### -None of this is done, although having it be a separate daemon sound weird. - -You would probably want to use ```twisted.mail.mail.FileMonitoringService``` to -watch the mailbox (is the mailbox virtual or a maildir or mbox or?) diff --git a/NOTES.md b/NOTES.md deleted file mode 100644 index a53f49d..0000000 --- a/NOTES.md +++ /dev/null @@ -1,59 +0,0 @@ - -# Questions # -------------- - -1. What is the lowest available RAM for a target server running a leap_mx? - 1.a. Do we want to store all id_keys and/or aliases in memory? - -2. Asked in discussion section of '''postfix pipeline''' on the [leap_mx wiki -page](https://we.riseup.net/leap/mx) : - - "What is the best way to have postfix write a message to a spool directory? - There is a built-in facility for saving to a maildir, so we could just - specify a common maildir for everyone. alternately, we could pipe to a - simple command that was responsible for safely saving the file to disk. a - third possibility would be to have a local long running daemon that spoke - lmtp that postfix forward the message on to for delivery." - - I think that maildir is fine, but perhaps this will slow things down more - than monitoring a spool file. I would also imagine that if the server is - supposed to stand up to high loads, a spool file I/O blocks with every - email added to the queue. - -3. How do get it to go faster? Should we create some mockups and benchmark -them? Could we attempt to learn which aliases are most often resolved and -prioritize keeping those in in-memory mappings? Is -[memcache](http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt) -a viable protocol for this, and how would it interact with CouchDB? - -4. What lib should we use for Python + Twisted + GPG/PGP ? - 4.a. It looks like most people are using python-gnupg... - - -## Tickets ## -------------- - -'''To be created:''' - -ticket for feature-alias_resolver_couchdb_support: - - o The alias resolver needs to speak to a couchdb/bigcouch - instance(s). Currently, it merely creates an in-memory dictionary - mapping. It seems like paisley is the best library for this. - -ticket for feature-check_recipient: - - o Need various errors for anything that could go wrong, e.g. the recipient - address is malformed, sender doesn't have permissions to send to such - address, etc. - o These errcodes need to follow the SMTP server transport code spec. - -ticket for feature-virtual_alias_map: - - o Get the recipient's userid from couchdb. - -ticket for feature-evaluate_python_gnupg: - - o Briefly audit library in order to assess if it has the necessary - features, as well as its general code quality. - diff --git a/bootstrap b/bootstrap deleted file mode 100755 index b76a572..0000000 --- a/bootstrap +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -############################################################################## -# -# bootstrap -# ----------------------- -# Setup a virtualenv, without ever using sudo. -# -# @author Isis Agora Lovecruft, 0x2cdb8b35 -# @date 18 February 2013 -# @version 0.0.1 -############################################################################## - -set -ex -- - -PYTHON=$(which python) -GIT=$(which git) - -VENV_VERSION=1.8.4 -VENV_URL=https://pypi.python.org/packages/source/v/virtualenv -VENV_TARBALL=virtualenv-${VENV_VERSION}.tar.gz - -VENV_WRAPPER_VERSION=3.6 -VENV_WRAPPER_URL=https://pypi.python.org/packages/source/v/virtualenvwrapper -VENV_WRAPPER_TARBALL=virtualenvwrapper-${VENV_WRAPPER_VERSION}.tar.gz - -BOOTSTRAP_ENV=.bootstrap -BOOTSTRAP_OPTS='--no-site-packages --setuptools --unzip-setuptools --never-download' - -PACKAGE_NAME=leap_mx -PACKAGE_URL=https://github.com/isislovecruft/leap_mx.git -PACKAGE_WORKON=${PWD}/${PACKAGE_NAME} -PACKAGE_REQUIREMENTS=${PACKAGE_WORKON}/pkg/mx-requirements.pip -PACKAGE_OPTS=${BOOTSTRAP_OPTS}'' ## xxx add parameter for extra options - -echo 'Downloading virtualenv source from' -echo "${VENV_URL}..." -\wget -O ${VENV_TARBALL} ${VENV_URL}/${VENV_TARBALL} -tar xvzf ${VENV_TARBALL} - -echo 'Downloading virtualenv-wrapper source from:' -echo "${VENV_WRAPPER_URL}" -\wget -O $VENV_WRAPPER_TARBALL ${VENV_WRAPPER_URL}/${VENV_WRAPPER_TARBALL} -tar xvzf virtualenvwrapper-${VENV_WRAPPER_VERSION}.tar.gz - - -echo 'Creating initial virtualenv bootstrap environment, called "bootstrap"' -echo 'in which we will install virtualenv, to avoid using sudo.' -$PYTHON virtualenv-${VENV_VERSION}/virtualenv.py $BOOTSTRAP_OPTS $BOOTSTRAP_ENV -rm -rf virtualenv-${VENV_VERSION} -${BOOTSTRAP_ENV}/bin/pip install ${VENV_TARBALL} -echo 'Installing virtualenvwrapper in "bootstrap" virtualenv...' -${BOOTSTRAP_ENV}/bin/pip install ${VENV_WRAPPER_TARBALL} - -echo 'Using "bootstrap" virtualenv to create project virtualenv...' -source ${BOOTSTRAP_ENV}/local/bin/virtualenvwrapper.sh -echo "Cloning from ${PACKAGE_URL}..." -${GIT} clone ${PACKAGE_URL} ${PACKAGE_NAME} -mkvirtualenv -a $PROJECT_WORKON -r ${PACKAGE_REQUIREMENTS} \ - ${PACKAGE_OPTS} ${PACKAGE_NAME} diff --git a/doc/DESIGN.md b/doc/DESIGN.md new file mode 100644 index 0000000..2d9fe82 --- /dev/null +++ b/doc/DESIGN.md @@ -0,0 +1,238 @@ +# design # + +## overview # +---------------------- +This page pertains to the incoming mail exchange servers of the provider. + +General overview of how incoming email will work: + + 1. Incoming message is received by provider's MX servers. + 2. The MTA (postfix in our case) does a ton of checks on the message before we + even check to see if the recipient is valid (this comes from experience + running the riseup mail infrastructure, where the vast majority of messages + can be rejected early in the SMTP reception and thus save a ton of processing + time on the server). + 3. Postfix then queries the database to check if the recipient is valid, if + they are over quota, if their account is enabled, and to resolve any aliases + for the account. + 4. The message is then delivered to an on-disk message spool. + 5. A daemon watches for new files in this spool. Each message is encrypted to + the user's public key, and stored in the user's incoming message queue (stored + in couchdb), and removed from disk. + 6. When the user next logs in with their client, the user's message queue is + emptied by the client. + 7. Each message is decrypted by the client, and then stored in the user's + "inbox" as an unread message. + 8. This local inbox uses soledad for storage + 9. Soledad, in the background, will then re-encrypt this email (now a soledad + document), and sync to the cloud. + +## postfix pipeline ## +--------------------------- +incoming mx servers will run postfix, configured in a particular way: + + 1. postscreen: before accepting an incoming message, checks RBLs, checks RFC + validity, checks for spam pipelining. + (pass) proceed to next step. + (fail) return SMTP error, which bounces email. + 2. more SMTP checks: valid hostnames, etc. + (pass) accepted, proceed to next step. + (fail) return SMTP error, which bounces email. + 3. check_recipient_access -- look up each recipient and ensure they are + allowed to receive messages. + (pass) empty result, proceed to next step. + (fail) return SMTP error code and error comment, bounce message. + 4. milter processessing (spamassassin & clamav) + (pass) continue + (fail) bounce message, flag as spam, or silently kill. + 5. virtual_alias_maps -- map user defined aliases and forwards + (local address) continue if new address is for this mx + (remote address) continue. normally, postfix would relay to the remote domain, but we don't want that. + 6. deliver message to spool + (write) save the message to disk on the mx. + 7. postfix's job is done, mail_receiver picks up email from spool directory + +Questions: + + * what is the best way to have postfix write a message to a spool directory? + There is a built-in facility for saving to a maildir, so we could just + specify a common maildir for everyone. alternately, we could pipe to a + simple command that was responsible for safely saving the file to disk. a + third possibility would be to have a local long running daemon that spoke + lmtp that postfix forward the message on to for delivery. + * if virtual_alias_maps comes after check_recipient_access, then a user with + aliases set but who is over quota will not be able to forward email. i think + this is fine. + * if we are going to support forwarding, we should ensure that the message + gets encrypted before getting forwarded. so, postfix should not do any + forwarding. instead, this should be the job of mail_receiver. + +Considerations: + + 1. high load should fill queue, not crash pipeline: It is important that the + pipeline be able to handle massive bursts of email, as often happens with + email. This means map lookups need to be very fast, and when there is a high + load of email postfix should not be waiting on the mail receiver but must be + able to pass the message off quickly and have the slower mail receiver churn + through the backlog as best it can. + 2. don't lose messages: It is important to not lose any messages when there is + a problem. So, generally, a copy of an email should always exist in some spool + somewhere, and that copy should not be deleted until there is confirmation + that the next stage has succeeded. + +## alias_resolver ## +------------------------------ +The alias_resolver will be a daemon running on MX servers that handles lookups +in the user database of email aliases, forwards, quota, and account status. + +Communication with: + + 1. postfix:: alias_resolver will be bound to localhost and speak postfix's + very simple [tcp map protocol -> http://www.postfix.org/tcp_table.5.html]. + + 2. couchdb:: alias_resolver will make couchdb queries to a local http load + balancer that connects to a couchdb/bigcouch + cluster. [directly accessing the couch->https://we.riseup.net/leap+platform/querying-the-couchdb] + might help getting started. + +### Discussion: ### + + 1. we want the lookups to be fast. using views in couchdb, these should be + very fast. when using bigcouch, we can make it faster by specifying a read + quorum of 1 (instead of the default 2). this will make it so that only a + single couchdb needs to be queried to find the result. i don't know if this + would cause problems, but aliases don't change very often. + +alias_resolver will be responsible for two map lookups in postfix: + +#### check_recipient #### +------------------------- +postfix config: + +@check_recipient_access tcp:localhost:1000@ + +postfix will send "get username@domain.org" and alias_resolver should return an +empty result ("200 \n", i think) if postfix should deliver email to the +user. otherwise, it should return an error. here is example response, verbatim, +that can be used to bounce over quota users: + +``` +200 DEFER_IF_PERMIT Sorry, your message cannot be delivered because the +recipient's mailbox is full. If you can contact them another way, you may wish +to tell them of this problem. +``` + +"DEFER_IF_PERMIT" will let the other MX know that this error is temporary and +that they should try again soon. Typically, an MX will try repeatedly, at +longer and longer intervals, for four days before giving up. + +#### virtual alias map #### +--------------------------- +postfix config: + +@virtual_alias_map tcp:localhost:1001@ + +postfix will send "get alias-address@domain.org" and alias_resolver should +return "200 id_123456\n", where 123456 is the unique id of the user that has +alias-address@domain.org. + +couchdb should have a view that will let us query on an (alias) address and +return the user id. + +note: if the result of the alias map (e.g. id_123456) does not have a domain +suffix, i think postfix will use the 'default transport'. if we want it to use +the virtual transport instead, we should append the domain (eg +id_123456@example.org). see +http://www.postfix.org/ADDRESS_REWRITING_README.html#resolve + + +### Current status: ### +The current implementation of alias_resolver is in +leap-mx/src/leap/mx/alias_resolver.py. + +The class ```alias_resolver.StatusCodes``` deals with creating SMTP-like +response messages for Postfix, speaking Postfix's TCP Map protocol (from item +#1). + +As for Discussion item #1: + +It might be possible to use +[python-memcached](https://pypi.python.org/pypi/python-memcached/) as an +interface to a [memcached](http://memcached.org/) instance to speed up database +lookups, by keeping an in memory mapping of recent request/response +pairs. Also, Twisted now (I think as of 12.0.0) ships with a protocol for +handling Memcached servers, this is in ```twisted.protocols.memcache```. This +should be prioritised for later, if it is decided that querying the CouchDB is +too expensive or time-consuming. + +Thus far, to speed up alias lookup, an in-memory mapping of alias<->resolution +pairs is created by ```alias_resolver.AliasResolverFactory()```, which can be +optionally seeded with a dictionary of ```{ 'alias': 'resolution' }``` pairs +by doing: +~~~~~~ +>>> from leap.mx import alias_resolver +>>> aliasResolverFactory = alias_resolver.AliasResolverFactory( +... addr='1.2.3.4', port=4242, data={'isis': 'isis@leap.se', +... 'drebs': 'drebs@leap.se'}) +>>> aliasResolver = aliasResolverFactory.buildProtocol() +>>> aliasResolver.check_recipient_access('isis') +200 OK Others might say 'HELLA AWESOME'...but we're not convinced. +~~~~~~ + +TODO: + 1. The AliasResolverFactory needs to be connected to the CouchDB. The + classmethod in which this should occur is ```AliasResolverFactory.get()```. + + 2. I am not sure where to get the user's UUID from (Soledad?). Wherever we get + it from, it will need to be returned in + ```AliasResolver.virtual_alias_map()```, and if we want Postfix to hear about + it, then that response will need to be fed into ```AliasResolver.sendCode```. + + 3. Other than those two things, I think everything is done. The only potential + other thing I can think of is that the codes in + ```alias_resolver.StatusCodes``` might need to be urlencoded for Postfix to + accept them, but this is like two lines of code from urllib. + + + +## mail_receiver ## + +the mail_receiver is a daemon that runs on incoming MX servers and is +responsible for encrypting incoming email to the user's public key and saving +the email to an incoming queue database for that user. + +communicates with: + + * message spool directory:: mail_reciever sits and waits for new email to be + written to the spool directory (maybe using this + https://github.com/seb-m/pyinotify, i think it is better than FAM). when a + new file is dumped into the spool, mail_receiver reads the file, encrypts + the entire thing using the public key of the recipient, and saves to + couchdb. + * couchdb get:: mail_receiver does a query on user id to get back user's + public openpgp key. read quorum of 1 is probably ok. + * couchdb put:: mail_receiver communicates with couchdb for storing encrypted + email for each user (eventually, mail_receiver will communicate with a local + http proxy, that communicates with a bigcouch cluster, but the api is + identical) + +discussion: + * i am not sure if postfix adds a header to indicate to whom a message was + actually delivered. if not, this is a problem, because then how do we know + what db to put it in or what public key to use? this is perhaps a good + reason to not let postfix handle writing the message to disk, but instead + pipe it to another command (because postfix sets env variables for stuff + like recipient). + + * should the incoming message queue be a separate database or should it be + just documents in the user's main database with special flags? + + * whenever possible, we should refer to the user by a fixed id, not their + username, because we want to support the ability to change usernames. so, + for example, database names should not be based on usernames. + +### Current Status: ### +None of this is done, although having it be a separate daemon sound weird. + +You would probably want to use ```twisted.mail.mail.FileMonitoringService``` to +watch the mailbox (is the mailbox virtual or a maildir or mbox or?) diff --git a/doc/NOTES.md b/doc/NOTES.md new file mode 100644 index 0000000..a53f49d --- /dev/null +++ b/doc/NOTES.md @@ -0,0 +1,59 @@ + +# Questions # +------------- + +1. What is the lowest available RAM for a target server running a leap_mx? + 1.a. Do we want to store all id_keys and/or aliases in memory? + +2. Asked in discussion section of '''postfix pipeline''' on the [leap_mx wiki +page](https://we.riseup.net/leap/mx) : + + "What is the best way to have postfix write a message to a spool directory? + There is a built-in facility for saving to a maildir, so we could just + specify a common maildir for everyone. alternately, we could pipe to a + simple command that was responsible for safely saving the file to disk. a + third possibility would be to have a local long running daemon that spoke + lmtp that postfix forward the message on to for delivery." + + I think that maildir is fine, but perhaps this will slow things down more + than monitoring a spool file. I would also imagine that if the server is + supposed to stand up to high loads, a spool file I/O blocks with every + email added to the queue. + +3. How do get it to go faster? Should we create some mockups and benchmark +them? Could we attempt to learn which aliases are most often resolved and +prioritize keeping those in in-memory mappings? Is +[memcache](http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt) +a viable protocol for this, and how would it interact with CouchDB? + +4. What lib should we use for Python + Twisted + GPG/PGP ? + 4.a. It looks like most people are using python-gnupg... + + +## Tickets ## +------------- + +'''To be created:''' + +ticket for feature-alias_resolver_couchdb_support: + + o The alias resolver needs to speak to a couchdb/bigcouch + instance(s). Currently, it merely creates an in-memory dictionary + mapping. It seems like paisley is the best library for this. + +ticket for feature-check_recipient: + + o Need various errors for anything that could go wrong, e.g. the recipient + address is malformed, sender doesn't have permissions to send to such + address, etc. + o These errcodes need to follow the SMTP server transport code spec. + +ticket for feature-virtual_alias_map: + + o Get the recipient's userid from couchdb. + +ticket for feature-evaluate_python_gnupg: + + o Briefly audit library in order to assess if it has the necessary + features, as well as its general code quality. + diff --git a/gpg-key-generator b/gpg-key-generator deleted file mode 100755 index 54d29fb..0000000 --- a/gpg-key-generator +++ /dev/null @@ -1,209 +0,0 @@ -#!/bin/bash -############################################################################## -# -# gpg-key-generator -# ------------------- -# Create batch processed keys for dummy users in the CouchDB, for testing. -# see -# www.gnupg.org/documentation/manuals/gnupg-devel/Unattended-GPG-key-generation.html -# for syntax specification. -# -# @author Isis Agora Lovecruft, 0x2cdb8b35 -# @date 11 February 2013 -# @version 0.1.0 -############################################################################## - - -here="${PWD}" -test_dir="${here}/gpg-keys" -batch_file="${test_dir}/gpg-batch-key-script" -default_keypair_password="leap" -default_keypair_email="blackhole@leap.se" - -function usage () -{ - echo -e "\033[40m\033[36m Usage: $0 [-n [-c|-a]] | [-d|-h]\033[0m" - echo -e "\033[40m\033[36m --------------------------------------------------------\033[0m" - echo -e "\033[40m\033[36m Creates a set of GPG dummy keys for unittesting purposes.\033[0m" - echo - echo -e "\033[40m\033[36m Keys will be created in ${test_dir}, \033[0m" - echo -e "\033[40m\033[36m and a GnuPG batch file named ${batch_file##*/} \033[0m" - echo -e "\033[40m\033[36m will also be created in that same directory. \033[0m" - echo -e "\033[40m\033[36m The default password to all keys is: "'"'"${default_keypair_passwd}"'"'" \033[0m" - echo - echo -e "\033[40m\033[36m Options:\033[0m" - echo -e "\033[40m\033[36m -n,--number Number of keys (to create/append) \033[0m" - echo -e "\033[40m\033[36m -c,--create Create a fresh set of N test keys \033[0m" - echo -e "\033[40m\033[36m -a,--append Append another set of N test keys \033[0m" - echo -e "\033[40m\033[36m -d,--delete Delete the test keys and directory\033[0m" - echo -e "\033[40m\033[36m -h,--help This cruft\033[0m" -} - -## @param $1: the filename to write to -## @param $2: the directory to place test keys and batch files in -## @param $3: the number of keypairs to create -function write_gpg_batch_file () -{ - ## if the test directory doesn't exist, create it: - if ! test -w "${1}" ; then - if ! test -d "${2}"; then - mkdir $2 - fi - fi - - # if the batch file is already there, ask to back it up: - if test -r "${1}" ; then - read -ers -N 1 -t 60 \ - -p"Should we keep a backup copy the previous batch file? (Y/n) " bak - case $bak in - n|N ) echo -e "\033[40m\033[31m Overwriting ${1}...\033[0m" ;; - * ) iii=0 - backup="${1}.${iii}-"$(date +"%F")".bak" - while ! test -r "$backup" ; do - echo -e"\033[40m\033[36m Backing up to: \033[0m" - echo -e"\033[40m\033[36m ${backup} \033[0m" - cp $1 $backup - iii=$(( $iii + 1 )) - done ;; - esac - ## then always delete the old otherwise we'll append to that and generate - ## the previous batch's keys too: - ! test -r "${1}" || rm $1 - fi - - ## and whether we backed up or not, make our file if it doesn't exist: - if ! test -w "${1}" ; then - touch $1 && chmod +rw $1 - fi - echo -e "\033[40m\033[36m Writing GPG key generation batch file to: \033[0m" - echo -e "\033[40m\033[36m ${1}... \033[0m" - - total_keypairs=$(printf "%03d" ${3}) - echo "Total keypairs to be generated: ${total_keypairs}" - - this_month=$(date +"%m") # ## this is awkward...isn't there - expire_soon=$(( ${this_month} + 1 )) ## a better way? - next_month=$(printf "%02d" ${expire_soon}) - expiry_date=$(date +"%Y-")${next_month}$(date +"-%d") - echo "Expiry date for keypairs: ${expiry_date}" - - for i in $(seq -f "%03g" 1 $3 ) ; do - now=$(date +"%Y-%m-%d_%H-%M") - echo "Writing generation parameters for keypair #${i}..." - cat >> $1 < 0 ]] ; then - SHORTS="hcadn:" - LONGS="help,create,append,destroy,number:" - ARGS=$(getopt -s bash --options $SHORTS --longoptions $LONGS \ - --name ${0##*/} -- "$@") - - if [ $? != 0 ] ; then - echo -e "\033[40m\033[31m Unable to parse options. \033[0m">&2 - exit 1 - fi - eval set -- "$ARGS" - while test -n "$1" ; do - case $1 in - -n|--number ) export CREATE_N="$2" - if test -z "$CREATE_N"; then CREATE_N="3"; fi; - shift 2 ;; - -c|--create ) delete_batch_keys ${test_dir} - write_gpg_batch_file ${batch_file} ${test_dir} \ - ${CREATE_N} - run_gpg_batch_file ${batch_file} ${test_dir} - shift ;; - -a|--append ) run_gpg_batch_file ${batch_file} ${test_dir} - shift ;; - -d|--destroy ) delete_batch_keys ${test_dir} ; shift ;; - --) shift ; break ;; - * ) usage ; shift ;; - esac - done - finish -else - usage -fi - -unset here test_dir batch_file CREATE_N - diff --git a/gpg-keys/generated-2013-02-15_19-15-001.pub b/gpg-keys/generated-2013-02-15_19-15-001.pub deleted file mode 100644 index 1c8fd34..0000000 Binary files a/gpg-keys/generated-2013-02-15_19-15-001.pub and /dev/null differ diff --git a/gpg-keys/generated-2013-02-15_19-15-001.sec b/gpg-keys/generated-2013-02-15_19-15-001.sec deleted file mode 100644 index 97a60e7..0000000 Binary files a/gpg-keys/generated-2013-02-15_19-15-001.sec and /dev/null differ diff --git a/gpg-keys/generated-2013-02-15_19-15-002.pub b/gpg-keys/generated-2013-02-15_19-15-002.pub deleted file mode 100644 index 1cbf8d8..0000000 Binary files a/gpg-keys/generated-2013-02-15_19-15-002.pub and /dev/null differ diff --git a/gpg-keys/generated-2013-02-15_19-15-002.sec b/gpg-keys/generated-2013-02-15_19-15-002.sec deleted file mode 100644 index f89cd97..0000000 Binary files a/gpg-keys/generated-2013-02-15_19-15-002.sec and /dev/null differ diff --git a/gpg-keys/generated-2013-02-15_19-15-003.pub b/gpg-keys/generated-2013-02-15_19-15-003.pub deleted file mode 100644 index bc0ac12..0000000 Binary files a/gpg-keys/generated-2013-02-15_19-15-003.pub and /dev/null differ diff --git a/gpg-keys/generated-2013-02-15_19-15-003.sec b/gpg-keys/generated-2013-02-15_19-15-003.sec deleted file mode 100644 index 397f44a..0000000 Binary files a/gpg-keys/generated-2013-02-15_19-15-003.sec and /dev/null differ diff --git a/gpg-keys/gpg-batch-key-script b/gpg-keys/gpg-batch-key-script deleted file mode 100644 index 7e48db0..0000000 --- a/gpg-keys/gpg-batch-key-script +++ /dev/null @@ -1,45 +0,0 @@ -%echo Generating keypair 001/003... -Key-Type: RSA -Key-Length: 4096 -Subkey-Type: RSA -Subkey-Length: 4096 -Name-Real: Louis Lingg -Name-Email: blackhole@leap.se -Name-Comment: Test Key 001/003 -Expire-Date: 2013-03-15 -Passphrase: leap -%pubring generated-2013-02-15_19-15-001.pub -%secring generated-2013-02-15_19-15-001.sec -%commit -%echo done. 001 keys out of 003 completed. - -%echo Generating keypair 002/003... -Key-Type: RSA -Key-Length: 4096 -Subkey-Type: RSA -Subkey-Length: 4096 -Name-Real: Louis Lingg -Name-Email: blackhole@leap.se -Name-Comment: Test Key 002/003 -Expire-Date: 2013-03-15 -Passphrase: leap -%pubring generated-2013-02-15_19-15-002.pub -%secring generated-2013-02-15_19-15-002.sec -%commit -%echo done. 002 keys out of 003 completed. - -%echo Generating keypair 003/003... -Key-Type: RSA -Key-Length: 4096 -Subkey-Type: RSA -Subkey-Length: 4096 -Name-Real: Louis Lingg -Name-Email: blackhole@leap.se -Name-Comment: Test Key 003/003 -Expire-Date: 2013-03-15 -Passphrase: leap -%pubring generated-2013-02-15_19-15-003.pub -%secring generated-2013-02-15_19-15-003.sec -%commit -%echo done. 003 keys out of 003 completed. - diff --git a/pkg/utils/bootstrap b/pkg/utils/bootstrap new file mode 100755 index 0000000..b76a572 --- /dev/null +++ b/pkg/utils/bootstrap @@ -0,0 +1,59 @@ +#!/bin/bash +############################################################################## +# +# bootstrap +# ----------------------- +# Setup a virtualenv, without ever using sudo. +# +# @author Isis Agora Lovecruft, 0x2cdb8b35 +# @date 18 February 2013 +# @version 0.0.1 +############################################################################## + +set -ex -- + +PYTHON=$(which python) +GIT=$(which git) + +VENV_VERSION=1.8.4 +VENV_URL=https://pypi.python.org/packages/source/v/virtualenv +VENV_TARBALL=virtualenv-${VENV_VERSION}.tar.gz + +VENV_WRAPPER_VERSION=3.6 +VENV_WRAPPER_URL=https://pypi.python.org/packages/source/v/virtualenvwrapper +VENV_WRAPPER_TARBALL=virtualenvwrapper-${VENV_WRAPPER_VERSION}.tar.gz + +BOOTSTRAP_ENV=.bootstrap +BOOTSTRAP_OPTS='--no-site-packages --setuptools --unzip-setuptools --never-download' + +PACKAGE_NAME=leap_mx +PACKAGE_URL=https://github.com/isislovecruft/leap_mx.git +PACKAGE_WORKON=${PWD}/${PACKAGE_NAME} +PACKAGE_REQUIREMENTS=${PACKAGE_WORKON}/pkg/mx-requirements.pip +PACKAGE_OPTS=${BOOTSTRAP_OPTS}'' ## xxx add parameter for extra options + +echo 'Downloading virtualenv source from' +echo "${VENV_URL}..." +\wget -O ${VENV_TARBALL} ${VENV_URL}/${VENV_TARBALL} +tar xvzf ${VENV_TARBALL} + +echo 'Downloading virtualenv-wrapper source from:' +echo "${VENV_WRAPPER_URL}" +\wget -O $VENV_WRAPPER_TARBALL ${VENV_WRAPPER_URL}/${VENV_WRAPPER_TARBALL} +tar xvzf virtualenvwrapper-${VENV_WRAPPER_VERSION}.tar.gz + + +echo 'Creating initial virtualenv bootstrap environment, called "bootstrap"' +echo 'in which we will install virtualenv, to avoid using sudo.' +$PYTHON virtualenv-${VENV_VERSION}/virtualenv.py $BOOTSTRAP_OPTS $BOOTSTRAP_ENV +rm -rf virtualenv-${VENV_VERSION} +${BOOTSTRAP_ENV}/bin/pip install ${VENV_TARBALL} +echo 'Installing virtualenvwrapper in "bootstrap" virtualenv...' +${BOOTSTRAP_ENV}/bin/pip install ${VENV_WRAPPER_TARBALL} + +echo 'Using "bootstrap" virtualenv to create project virtualenv...' +source ${BOOTSTRAP_ENV}/local/bin/virtualenvwrapper.sh +echo "Cloning from ${PACKAGE_URL}..." +${GIT} clone ${PACKAGE_URL} ${PACKAGE_NAME} +mkvirtualenv -a $PROJECT_WORKON -r ${PACKAGE_REQUIREMENTS} \ + ${PACKAGE_OPTS} ${PACKAGE_NAME} diff --git a/pkg/utils/gpg-key-generator b/pkg/utils/gpg-key-generator new file mode 100755 index 0000000..54d29fb --- /dev/null +++ b/pkg/utils/gpg-key-generator @@ -0,0 +1,209 @@ +#!/bin/bash +############################################################################## +# +# gpg-key-generator +# ------------------- +# Create batch processed keys for dummy users in the CouchDB, for testing. +# see +# www.gnupg.org/documentation/manuals/gnupg-devel/Unattended-GPG-key-generation.html +# for syntax specification. +# +# @author Isis Agora Lovecruft, 0x2cdb8b35 +# @date 11 February 2013 +# @version 0.1.0 +############################################################################## + + +here="${PWD}" +test_dir="${here}/gpg-keys" +batch_file="${test_dir}/gpg-batch-key-script" +default_keypair_password="leap" +default_keypair_email="blackhole@leap.se" + +function usage () +{ + echo -e "\033[40m\033[36m Usage: $0 [-n [-c|-a]] | [-d|-h]\033[0m" + echo -e "\033[40m\033[36m --------------------------------------------------------\033[0m" + echo -e "\033[40m\033[36m Creates a set of GPG dummy keys for unittesting purposes.\033[0m" + echo + echo -e "\033[40m\033[36m Keys will be created in ${test_dir}, \033[0m" + echo -e "\033[40m\033[36m and a GnuPG batch file named ${batch_file##*/} \033[0m" + echo -e "\033[40m\033[36m will also be created in that same directory. \033[0m" + echo -e "\033[40m\033[36m The default password to all keys is: "'"'"${default_keypair_passwd}"'"'" \033[0m" + echo + echo -e "\033[40m\033[36m Options:\033[0m" + echo -e "\033[40m\033[36m -n,--number Number of keys (to create/append) \033[0m" + echo -e "\033[40m\033[36m -c,--create Create a fresh set of N test keys \033[0m" + echo -e "\033[40m\033[36m -a,--append Append another set of N test keys \033[0m" + echo -e "\033[40m\033[36m -d,--delete Delete the test keys and directory\033[0m" + echo -e "\033[40m\033[36m -h,--help This cruft\033[0m" +} + +## @param $1: the filename to write to +## @param $2: the directory to place test keys and batch files in +## @param $3: the number of keypairs to create +function write_gpg_batch_file () +{ + ## if the test directory doesn't exist, create it: + if ! test -w "${1}" ; then + if ! test -d "${2}"; then + mkdir $2 + fi + fi + + # if the batch file is already there, ask to back it up: + if test -r "${1}" ; then + read -ers -N 1 -t 60 \ + -p"Should we keep a backup copy the previous batch file? (Y/n) " bak + case $bak in + n|N ) echo -e "\033[40m\033[31m Overwriting ${1}...\033[0m" ;; + * ) iii=0 + backup="${1}.${iii}-"$(date +"%F")".bak" + while ! test -r "$backup" ; do + echo -e"\033[40m\033[36m Backing up to: \033[0m" + echo -e"\033[40m\033[36m ${backup} \033[0m" + cp $1 $backup + iii=$(( $iii + 1 )) + done ;; + esac + ## then always delete the old otherwise we'll append to that and generate + ## the previous batch's keys too: + ! test -r "${1}" || rm $1 + fi + + ## and whether we backed up or not, make our file if it doesn't exist: + if ! test -w "${1}" ; then + touch $1 && chmod +rw $1 + fi + echo -e "\033[40m\033[36m Writing GPG key generation batch file to: \033[0m" + echo -e "\033[40m\033[36m ${1}... \033[0m" + + total_keypairs=$(printf "%03d" ${3}) + echo "Total keypairs to be generated: ${total_keypairs}" + + this_month=$(date +"%m") # ## this is awkward...isn't there + expire_soon=$(( ${this_month} + 1 )) ## a better way? + next_month=$(printf "%02d" ${expire_soon}) + expiry_date=$(date +"%Y-")${next_month}$(date +"-%d") + echo "Expiry date for keypairs: ${expiry_date}" + + for i in $(seq -f "%03g" 1 $3 ) ; do + now=$(date +"%Y-%m-%d_%H-%M") + echo "Writing generation parameters for keypair #${i}..." + cat >> $1 < 0 ]] ; then + SHORTS="hcadn:" + LONGS="help,create,append,destroy,number:" + ARGS=$(getopt -s bash --options $SHORTS --longoptions $LONGS \ + --name ${0##*/} -- "$@") + + if [ $? != 0 ] ; then + echo -e "\033[40m\033[31m Unable to parse options. \033[0m">&2 + exit 1 + fi + eval set -- "$ARGS" + while test -n "$1" ; do + case $1 in + -n|--number ) export CREATE_N="$2" + if test -z "$CREATE_N"; then CREATE_N="3"; fi; + shift 2 ;; + -c|--create ) delete_batch_keys ${test_dir} + write_gpg_batch_file ${batch_file} ${test_dir} \ + ${CREATE_N} + run_gpg_batch_file ${batch_file} ${test_dir} + shift ;; + -a|--append ) run_gpg_batch_file ${batch_file} ${test_dir} + shift ;; + -d|--destroy ) delete_batch_keys ${test_dir} ; shift ;; + --) shift ; break ;; + * ) usage ; shift ;; + esac + done + finish +else + usage +fi + +unset here test_dir batch_file CREATE_N + diff --git a/pkg/utils/gpg-keys/generated-2013-02-15_19-15-001.pub b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-001.pub new file mode 100644 index 0000000..1c8fd34 Binary files /dev/null and b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-001.pub differ diff --git a/pkg/utils/gpg-keys/generated-2013-02-15_19-15-001.sec b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-001.sec new file mode 100644 index 0000000..97a60e7 Binary files /dev/null and b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-001.sec differ diff --git a/pkg/utils/gpg-keys/generated-2013-02-15_19-15-002.pub b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-002.pub new file mode 100644 index 0000000..1cbf8d8 Binary files /dev/null and b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-002.pub differ diff --git a/pkg/utils/gpg-keys/generated-2013-02-15_19-15-002.sec b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-002.sec new file mode 100644 index 0000000..f89cd97 Binary files /dev/null and b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-002.sec differ diff --git a/pkg/utils/gpg-keys/generated-2013-02-15_19-15-003.pub b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-003.pub new file mode 100644 index 0000000..bc0ac12 Binary files /dev/null and b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-003.pub differ diff --git a/pkg/utils/gpg-keys/generated-2013-02-15_19-15-003.sec b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-003.sec new file mode 100644 index 0000000..397f44a Binary files /dev/null and b/pkg/utils/gpg-keys/generated-2013-02-15_19-15-003.sec differ diff --git a/pkg/utils/gpg-keys/gpg-batch-key-script b/pkg/utils/gpg-keys/gpg-batch-key-script new file mode 100644 index 0000000..7e48db0 --- /dev/null +++ b/pkg/utils/gpg-keys/gpg-batch-key-script @@ -0,0 +1,45 @@ +%echo Generating keypair 001/003... +Key-Type: RSA +Key-Length: 4096 +Subkey-Type: RSA +Subkey-Length: 4096 +Name-Real: Louis Lingg +Name-Email: blackhole@leap.se +Name-Comment: Test Key 001/003 +Expire-Date: 2013-03-15 +Passphrase: leap +%pubring generated-2013-02-15_19-15-001.pub +%secring generated-2013-02-15_19-15-001.sec +%commit +%echo done. 001 keys out of 003 completed. + +%echo Generating keypair 002/003... +Key-Type: RSA +Key-Length: 4096 +Subkey-Type: RSA +Subkey-Length: 4096 +Name-Real: Louis Lingg +Name-Email: blackhole@leap.se +Name-Comment: Test Key 002/003 +Expire-Date: 2013-03-15 +Passphrase: leap +%pubring generated-2013-02-15_19-15-002.pub +%secring generated-2013-02-15_19-15-002.sec +%commit +%echo done. 002 keys out of 003 completed. + +%echo Generating keypair 003/003... +Key-Type: RSA +Key-Length: 4096 +Subkey-Type: RSA +Subkey-Length: 4096 +Name-Real: Louis Lingg +Name-Email: blackhole@leap.se +Name-Comment: Test Key 003/003 +Expire-Date: 2013-03-15 +Passphrase: leap +%pubring generated-2013-02-15_19-15-003.pub +%secring generated-2013-02-15_19-15-003.sec +%commit +%echo done. 003 keys out of 003 completed. + diff --git a/pkg/utils/test_bootstrap b/pkg/utils/test_bootstrap new file mode 100755 index 0000000..f072d10 --- /dev/null +++ b/pkg/utils/test_bootstrap @@ -0,0 +1,39 @@ +#!/bin/bash +############################################################################## +# +# test_bootstrap +# -------------- +# Test that the bootstrap script works correctly by making a temporary new +# user. +# +# @author Isis Agora Lovecruft, 0x2cdb8b35 +# @date 18 February 2013 +# @version 0.0.1 +############################################################################## + +set -ex - + +HERE=$(pwd) +TEST_USER=bootstraptester + +echo "Creating new user: "'"'"${TEST_USER}"'"'"" +sudo adduser --home /home/${TEST_USER} --shell /bin/bash ${TEST_USER} && \ + echo -e "notsecure\nnotsecure\n" | sudo passwd ${TEST_USER} + +echo 'Copying boostrap script to new user home directory...' +sudo cp ${HERE}/bootstrap /home/${TEST_USER}/bootstrap && \ + sudo chown ${TEST_USER}:${TEST_USER} /home/${TEST_USER}/bootstrap + +echo 'Logging in as new user and executing bootstrap script...' +echo 'Executing test of bootstrap script...' +## -S pulls password from stdin +echo -e "notsecure\n" | sudo -S -H -u ${TEST_USER} -i /home/${TEST_USER}/bootstrap + +if [[ "$?" != 0 ]] ; then + echo 'Error while testing bootstrap...' +else + echo 'Test of bootstrap successful.' +fi + +echo "Deleting user: "'"'"${TEST_USER}"'"'"" +sudo deluser --remove-home ${TEST_USER} diff --git a/sample-config/mx.conf b/sample-config/mx.conf new file mode 100644 index 0000000..2036706 --- /dev/null +++ b/sample-config/mx.conf @@ -0,0 +1,9 @@ +[mail1] +path=/home/blabla/Maildir/ +recursive=True + +[couchdb] +user=someuser +password=somepass +server=localhost +port=6666 diff --git a/test_bootstrap b/test_bootstrap deleted file mode 100755 index f072d10..0000000 --- a/test_bootstrap +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -############################################################################## -# -# test_bootstrap -# -------------- -# Test that the bootstrap script works correctly by making a temporary new -# user. -# -# @author Isis Agora Lovecruft, 0x2cdb8b35 -# @date 18 February 2013 -# @version 0.0.1 -############################################################################## - -set -ex - - -HERE=$(pwd) -TEST_USER=bootstraptester - -echo "Creating new user: "'"'"${TEST_USER}"'"'"" -sudo adduser --home /home/${TEST_USER} --shell /bin/bash ${TEST_USER} && \ - echo -e "notsecure\nnotsecure\n" | sudo passwd ${TEST_USER} - -echo 'Copying boostrap script to new user home directory...' -sudo cp ${HERE}/bootstrap /home/${TEST_USER}/bootstrap && \ - sudo chown ${TEST_USER}:${TEST_USER} /home/${TEST_USER}/bootstrap - -echo 'Logging in as new user and executing bootstrap script...' -echo 'Executing test of bootstrap script...' -## -S pulls password from stdin -echo -e "notsecure\n" | sudo -S -H -u ${TEST_USER} -i /home/${TEST_USER}/bootstrap - -if [[ "$?" != 0 ]] ; then - echo 'Error while testing bootstrap...' -else - echo 'Test of bootstrap successful.' -fi - -echo "Deleting user: "'"'"${TEST_USER}"'"'"" -sudo deluser --remove-home ${TEST_USER} -- cgit v1.2.3 From 23fbf0eabb6563b05a7264dff63af7fbe64cfdff Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 25 Apr 2013 13:01:01 -0300 Subject: Simplify check recipient access, change the owner field in mail_receiver --- src/leap/mx/alias_resolver.py | 4 ++++ src/leap/mx/check_recipient_access.py | 26 ++++---------------------- src/leap/mx/mail_receiver.py | 4 +++- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/src/leap/mx/alias_resolver.py b/src/leap/mx/alias_resolver.py index 8155ff9..3f93f0d 100644 --- a/src/leap/mx/alias_resolver.py +++ b/src/leap/mx/alias_resolver.py @@ -53,6 +53,10 @@ class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory): orig_key = key try: logger.debug("Processing key: %s" % (key,)) + if key.find("@") == -1: + logger.debug("Ignoring key since it's not an email address") + return None + key = key.split("@")[0] key = key.split("+")[0] logger.debug("Final key to query: %s" % (key,)) diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py index 8d03297..3ea6e91 100644 --- a/src/leap/mx/check_recipient_access.py +++ b/src/leap/mx/check_recipient_access.py @@ -22,11 +22,9 @@ Classes for resolving postfix recipient access import logging -try: - from twisted.protocols import postfix -except ImportError: - print "This software requires Twisted. Please see the README file" - print "for instructions on getting required dependencies." +from twisted.protocols import postfix + +from leap.mx.alias_resolver import AliasResolverFactory logger = logging.getLogger(__name__) @@ -39,21 +37,5 @@ class CheckRecipientAccess(postfix.PostfixTCPMapServer): self.sendCode(200) -class CheckRecipientAccessFactory(postfix.PostfixTCPMapDeferringDictServerFactory): - +class CheckRecipientAccessFactory(AliasResolverFactory): protocol = CheckRecipientAccess - - def __init__(self, couchdb, *args, **kwargs): - postfix.PostfixTCPMapDeferringDictServerFactory.__init__(self, *args, **kwargs) - self._cdb = couchdb - - def get(self, key): - orig_key = key - try: - key = key.split("@")[0] - key = key.split("+")[0] - except Exception as e: - key = orig_key - logger.exception("%s" % (e,)) - d = self._cdb.queryByLoginOrAlias(key) - return d diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index ae32f25..9006471 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -105,7 +105,9 @@ def _process_incoming_email(users_db, mail_couchdb_url_prefix, self, filepath, m with filepath.open("r") as f: mail_data = f.read() mail = message_from_string(mail_data) - owner = mail["Delivered-To"] + owner = mail["To"] + if owner is None: # default to Delivered-To + owner = mail["Delivered-To"] owner = owner.split("@")[0] owner = owner.split("+")[0] logger.debug("Mail owner: %s" % (owner,)) -- cgit v1.2.3 From dd165dcbab7e686a238655fc68a5d7d472ee1736 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 25 Apr 2013 13:06:34 -0300 Subject: Update requirements --- pkg/mx-requirements.pip | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/mx-requirements.pip b/pkg/mx-requirements.pip index eda67e9..a05ac98 100644 --- a/pkg/mx-requirements.pip +++ b/pkg/mx-requirements.pip @@ -1,7 +1,4 @@ -Twisted==12.2.0 +Twisted paisley>=0.3.1 -PyYAML>=3.10 -pyxdg>=0.19-5 ## xxx change me to whatever you name the package in pypi -#python-gnupg>=0.3.0 -ipaddr \ No newline at end of file +python-gnupg>=0.3.0 -- cgit v1.2.3 From 4ed3e03aa4012905d04d62c1d34e3440f8e0dacc Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 25 Apr 2013 13:20:01 -0300 Subject: Add logfile capabilities --- start_mx.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/start_mx.py b/start_mx.py index 7e154da..bf3dddb 100755 --- a/start_mx.py +++ b/start_mx.py @@ -55,6 +55,7 @@ if __name__ == "__main__": debug = opts.debug config_file = opts.config + logfile = opts.log_file if debug: level = logging.DEBUG @@ -73,6 +74,13 @@ if __name__ == "__main__": console.setFormatter(formatter) logger.addHandler(console) + if logfile is not None: + logger.debug('Setting logfile to %s ', logfile) + fileh = logging.FileHandler(logfile) + fileh.setLevel(logging.DEBUG) + fileh.setFormatter(formatter) + logger.addHandler(fileh) + logger.info("~~~~~~~~~~~~~~~~~~~") logger.info(" LEAP MX") logger.info("~~~~~~~~~~~~~~~~~~~") -- cgit v1.2.3 From f6d6ff0862052605a5a3af328daadc0b8ce9e3bb Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 25 Apr 2013 15:25:07 -0300 Subject: Save unencrypted if no pubkey --- src/leap/mx/mail_receiver.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 9006471..20b5f4a 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -50,13 +50,15 @@ def _encrypt_message(uuid_pubkey, address_message): logger.debug("Encrypting message to %s's pubkey" % (uuid,)) logger.debug("Pubkey: %s" % (pubkey,)) - if pubkey is None or len(pubkey) == 0: - logger.exception("No public key found") - raise Exception("No public key found") - doc = LeapDocument(encryption_scheme=EncryptionSchemes.PUBKEY, doc_id=str(pyuuid.uuid4())) + if pubkey is None or len(pubkey) == 0: + doc.content = { + "_unencrypted_json": message + } + return uuid, doc + def _ascii_to_openpgp_cb(gpg): key = gpg.list_keys().pop() return openpgp._build_key_from_gpg(address, key, pubkey) -- cgit v1.2.3 From 9042a0f5d7d6cee5bacf8085328a7a29ae99df11 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 25 Apr 2013 15:31:09 -0300 Subject: Add incoming tag to unencrypted data too --- src/leap/mx/mail_receiver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 20b5f4a..00d93ba 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -53,9 +53,11 @@ def _encrypt_message(uuid_pubkey, address_message): doc = LeapDocument(encryption_scheme=EncryptionSchemes.PUBKEY, doc_id=str(pyuuid.uuid4())) + data = {'incoming': True, 'content': message} + if pubkey is None or len(pubkey) == 0: doc.content = { - "_unencrypted_json": message + "_unencrypted_json": json.dumps(data) } return uuid, doc @@ -65,8 +67,6 @@ def _encrypt_message(uuid_pubkey, address_message): openpgp_key = openpgp._safe_call(_ascii_to_openpgp_cb, pubkey) - data = {'incoming': True, 'content': message} - doc.content = { "_encrypted_json": openpgp.encrypt_asym(json.dumps(data), openpgp_key) } -- cgit v1.2.3 From 860329613b205708dc7a4b04bcaf39b9bd24fec6 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Thu, 2 May 2013 13:44:08 -0300 Subject: Several fixes according to the review comments --- mx.tac | 85 +++++++++ src/leap/mx/alias_resolver.py | 28 ++- src/leap/mx/check_recipient_access.py | 4 - src/leap/mx/couchdbhelper.py | 17 +- src/leap/mx/mail_receiver.py | 341 +++++++++++++++++----------------- start_mx.py | 134 ------------- 6 files changed, 277 insertions(+), 332 deletions(-) create mode 100755 mx.tac delete mode 100755 start_mx.py diff --git a/mx.tac b/mx.tac new file mode 100755 index 0000000..30ede8f --- /dev/null +++ b/mx.tac @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# start_mx.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +import ConfigParser + +from functools import partial + +from leap.mx import couchdbhelper +from leap.mx.mail_receiver import MailReceiver +from leap.mx.alias_resolver import AliasResolverFactory +from leap.mx.check_recipient_access import CheckRecipientAccessFactory + +try: + from twisted.application import service, internet + from twisted.internet import inotify + from twisted.internet.endpoints import TCP4ServerEndpoint + from twisted.python import filepath, log + from twisted.python import usage +except ImportError, ie: + print "This software requires Twisted>=12.0.2, please see the README for" + print "help on using virtualenv and pip to obtain requirements." + +config_file = "/etc/leap/mx.conf" + +config = ConfigParser.ConfigParser() +config.read(config_file) + +user = config.get("couchdb", "user") +password = config.get("couchdb", "password") + +server = config.get("couchdb", "server") +port = config.get("couchdb", "port") + +alias_port = config.getint("alias map", "port") +check_recipient_port = config.getint("check recipient", "port") + +cdb = couchdbhelper.ConnectedCouchDB(server, + port=port, + dbName="users", + username=user, + password=password) + + +application = service.Application("LEAP MX") + +# Alias map +alias_map = internet.TCPServer(alias_port, AliasResolverFactory(couchdb=cdb)) +alias_map.setServiceParent(application) + +# Check recipient access +check_recipient = internet.TCPServer(check_recipient_port, + CheckRecipientAccessFactory(couchdb=cdb)) +check_recipient.setServiceParent(application) + +# Mail receiver +mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user, + password, + server, + port) +directories = [] +for section in config.sections(): + if section in ("couchdb", "alias map", "check recipient"): + continue + to_watch = config.get(section, "path") + recursive = config.getboolean(section, "recursive") + directories.append([to_watch, recursive]) + +mr = MailReceiver(mail_couch_url_prefix, cdb, directories) +mr.setServiceParent(application) diff --git a/src/leap/mx/alias_resolver.py b/src/leap/mx/alias_resolver.py index 3f93f0d..68c6212 100644 --- a/src/leap/mx/alias_resolver.py +++ b/src/leap/mx/alias_resolver.py @@ -24,18 +24,15 @@ TODO: controlling concurrent connections and throttling resource consumption. """ -import logging - try: # TODO: we should probably use the system alias somehow # from twisted.mail import alias from twisted.protocols import postfix + from twisted.python import log except ImportError: print "This software requires Twisted. Please see the README file" print "for instructions on getting required dependencies." -logger = logging.getLogger(__name__) - class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory): def __init__(self, couchdb, *args, **kwargs): @@ -46,23 +43,24 @@ class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory): if isinstance(result, unicode): result = result.encode("utf8") if result is None: - logger.debug("Result not found") + log.msg("Result not found") return result def get(self, key): - orig_key = key try: - logger.debug("Processing key: %s" % (key,)) + log.msg("Processing key: %s" % (key,)) if key.find("@") == -1: - logger.debug("Ignoring key since it's not an email address") + log.msg("Ignoring key since it's not an email address") return None key = key.split("@")[0] key = key.split("+")[0] - logger.debug("Final key to query: %s" % (key,)) - except Exception as e: - key = orig_key - logger.exception("%s" % (e,)) - d = self._cdb.queryByLoginOrAlias(key) - d.addCallback(self._to_str) - return d + log.msg("Final key to query: %s" % (key,)) + d = self._cdb.queryByLoginOrAlias(key) + d.addCallback(self._to_str) + d.addErrback(log.err) + return d + except: + log.err() + + return None diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py index 3ea6e91..1b44504 100644 --- a/src/leap/mx/check_recipient_access.py +++ b/src/leap/mx/check_recipient_access.py @@ -20,14 +20,10 @@ Classes for resolving postfix recipient access """ -import logging - from twisted.protocols import postfix from leap.mx.alias_resolver import AliasResolverFactory -logger = logging.getLogger(__name__) - class CheckRecipientAccess(postfix.PostfixTCPMapServer): def _cbGot(self, value): diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index 2f6b548..7c4c8ce 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -20,8 +20,6 @@ Classes for working with CouchDB or BigCouch instances which store email alias maps, user UUIDs, and GPG keyIDs. """ -import logging - from functools import partial try: @@ -32,12 +30,11 @@ except ImportError: try: from twisted.internet import defer + from twisted.python import log except ImportError: print "This software requires Twisted. Please see the README file" print "for instructions on getting required dependencies." -logger = logging.getLogger(__name__) - class ConnectedCouchDB(client.CouchDB): """ @@ -84,9 +81,9 @@ class ConnectedCouchDB(client.CouchDB): @param data: response from the listDB command @type data: array """ - logger.msg("Available databases:") + log.msg("Available databases:") for database in data: - logger.msg(" * %s" % (database,)) + log.msg(" * %s" % (database,)) def createDB(self, dbName): """ @@ -119,7 +116,7 @@ class ConnectedCouchDB(client.CouchDB): reduce=False, include_docs=True) - d.addCallbacks(partial(self._get_uuid, alias), logger.error) + d.addCallbacks(partial(self._get_uuid, alias), log.err) return d @@ -138,14 +135,10 @@ class ConnectedCouchDB(client.CouchDB): for row in result["rows"]: if row["key"] == alias: uuid = row["id"] - try: - self._cache[uuid] = row["doc"]["public_key"] - except: - pass # no public key for this user + self._cache[uuid] = row["doc"].get("public_key", None) return uuid return None - def getPubKey(self, uuid): pubkey = None try: diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 00d93ba..2494a21 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -16,190 +16,197 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +""" +MailReceiver service definition +""" + import os import uuid as pyuuid -import logging -import argparse -import ConfigParser import json from email import message_from_string -from functools import partial - -from twisted.internet import inotify, reactor -from twisted.python import filepath -from leap.mx import couchdbhelper +from twisted.application.service import Service +from twisted.internet import inotify +from twisted.python import filepath, log from leap.soledad import LeapDocument from leap.soledad.backends.leap_backend import EncryptionSchemes from leap.soledad.backends.couch import CouchDatabase from leap.common.keymanager import openpgp -logger = logging.getLogger(__name__) - - -def _get_pubkey(uuid, cdb): - logger.debug("Fetching pubkey for %s" % (uuid,)) - return uuid, cdb.getPubKey(uuid) -def _encrypt_message(uuid_pubkey, address_message): - uuid, pubkey = uuid_pubkey - address, message = address_message - logger.debug("Encrypting message to %s's pubkey" % (uuid,)) - logger.debug("Pubkey: %s" % (pubkey,)) +class MailReceiver(Service): + """ + Service that monitors incoming email and processes it + """ + + def __init__(self, mail_couch_url, users_cdb, directories): + """ + Constructor + + @param mail_couch_url: URL prefix for the couchdb where mail + should be stored + @type mail_couch_url: str + @param users_cdb: CouchDB instance from where to get the uuid + and pubkey for a user + @type users_cdb: ConnectedCouchDB + @param directories: list of directories to monitor + @type directories: list of tuples (path: str, recursive: bool) + """ + # Service doesn't define an __init__ + self._mail_couch_url = mail_couch_url + self._users_cdb = users_cdb + self._directories = directories + + def startService(self): + """ + Starts the MailReceiver service + """ + Service.startService(self) + wm = inotify.INotify() + wm.startReading() + + mask = inotify.IN_CREATE + + for directory, recursive in self._directories: + log.msg("Watching %s --- Recursive: %s" % (directory, recursive)) + wm.watch(filepath.FilePath(directory), mask, callbacks=[self._process_incoming_email], recursive=recursive) + + def _get_pubkey(self, uuid): + """ + Given a UUID for a user, retrieve its public key + + @param uuid: UUID for a user + @type uuid: str + + @return: uuid, public key + @rtype: tuple of (str, str) + """ + log.msg("Fetching pubkey for %s" % (uuid,)) + return uuid, self._users_cdb.getPubKey(uuid) + + def _encrypt_message(self, uuid_pubkey, address, message): + """ + Given a UUID, a public key, address and a message, it encrypts + the message to that public key. + The address is needed in order to build the OpenPGPKey object. + + @param uuid_pubkey: tuple that holds the uuid and the public + key as it is returned by the previous call in the chain + @type uuid_pubkey: tuple (str, str) + @param address: mail address for this message + @type address: str + @param message: message contents + @type message: str + + @return: uuid, doc to sync with Soledad + @rtype: tuple(str, LeapDocument) + """ + uuid, pubkey = uuid_pubkey + log.msg("Encrypting message to %s's pubkey" % (uuid,)) + log.msg("Pubkey: %s" % (pubkey,)) + + doc = LeapDocument(doc_id=str(pyuuid.uuid4())) + + data = {'incoming': True, 'content': message} + + if pubkey is None or len(pubkey) == 0: + doc.content = { + "_encryption_scheme": EncryptionSchemes.NONE, + "_unencrypted_json": json.dumps(data) + } + return uuid, doc + + def _ascii_to_openpgp_cb(gpg): + key = gpg.list_keys().pop() + return openpgp._build_key_from_gpg(address, key, pubkey) + + openpgp_key = openpgp._safe_call(_ascii_to_openpgp_cb, pubkey) - doc = LeapDocument(encryption_scheme=EncryptionSchemes.PUBKEY, - doc_id=str(pyuuid.uuid4())) - - data = {'incoming': True, 'content': message} - - if pubkey is None or len(pubkey) == 0: doc.content = { - "_unencrypted_json": json.dumps(data) + "_encryption_scheme": EncryptionSchemes.PUBKEY, + "_encrypted_json": openpgp.encrypt_asym(json.dumps(data), openpgp_key) } - return uuid, doc - - def _ascii_to_openpgp_cb(gpg): - key = gpg.list_keys().pop() - return openpgp._build_key_from_gpg(address, key, pubkey) - - openpgp_key = openpgp._safe_call(_ascii_to_openpgp_cb, pubkey) - - doc.content = { - "_encrypted_json": openpgp.encrypt_asym(json.dumps(data), openpgp_key) - } - - return uuid, doc - - -def _export_message(uuid_doc, couch_url): - uuid, doc = uuid_doc - logger.debug("Exporting message for %s" % (uuid,)) - - if uuid is None: - uuid = 0 - - db = CouchDatabase(couch_url, "user-%s" % (uuid,)) - db.put_doc(doc) - logger.debug("Done exporting") - - return True - - -def _conditional_remove(do_remove, filepath): - if do_remove: - # remove the original mail - try: - logger.debug("Removing %s" % (filepath.path,)) - filepath.remove() - logger.debug("Done removing") - except Exception as e: - # TODO: better handle exceptions - logger.exception("%s" % (e,)) - - -def _process_incoming_email(users_db, mail_couchdb_url_prefix, self, filepath, mask): - if os.path.split(filepath.dirname())[-1] == "new": - logger.debug("Processing new mail at %s" % (filepath.path,)) - with filepath.open("r") as f: - mail_data = f.read() - mail = message_from_string(mail_data) - owner = mail["To"] - if owner is None: # default to Delivered-To - owner = mail["Delivered-To"] - owner = owner.split("@")[0] - owner = owner.split("+")[0] - logger.debug("Mail owner: %s" % (owner,)) - - logger.debug("%s received a new mail" % (owner,)) - d = users_db.queryByLoginOrAlias(owner) - d.addCallback(_get_pubkey, (users_db)) - d.addCallback(_encrypt_message, (owner, mail_data)) - d.addCallback(_export_message, (mail_couchdb_url_prefix)) - d.addCallback(_conditional_remove, (filepath)) - - -def main(): - epilog = "Copyright 2012 The LEAP Encryption Access Project" - parser = argparse.ArgumentParser(description="""LEAP MX Mail receiver""", epilog=epilog) - parser.add_argument('-d', '--debug', action="store_true", - help="Launches the LEAP MX mail receiver with debug output") - parser.add_argument('-l', '--logfile', metavar="LOG FILE", nargs='?', - action="store", dest="log_file", - help="Writes the logs to the specified file") - parser.add_argument('-c', '--config', metavar="CONFIG FILE", nargs='?', - action="store", dest="config", - help="Where to look for the configuration file. " \ - "Default: mail_receiver.cfg") - - opts, _ = parser.parse_known_args() - - debug = opts.debug - config_file = opts.config - - if debug: - level = logging.DEBUG - else: - level = logging.WARNING - - if config_file is None: - config_file = "leap_mx.cfg" - - logger.setLevel(level) - console = logging.StreamHandler() - console.setLevel(level) - formatter = logging.Formatter( - '%(asctime)s ' - '- %(name)s - %(levelname)s - %(message)s') - console.setFormatter(formatter) - logger.addHandler(console) + return uuid, doc - logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - logger.info(" LEAP MX Mail receiver") - logger.info("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") + def _export_message(self, uuid_doc): + """ + Given a UUID and a LeapDocument, it saves it directly in the + couchdb that serves as a backend for Soledad, in a db + accessible to the recipient of the mail + + @param uuid_doc: tuple that holds the UUID and LeapDocument + @type uuid_doc: tuple(str, LeapDocument) + + @return: True if it's ok to remove the message, False + otherwise + @rtype: bool + """ + uuid, doc = uuid_doc + log.msg("Exporting message for %s" % (uuid,)) + + if uuid is None: + uuid = 0 + + db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,)) + db.put_doc(doc) + + log.msg("Done exporting") + + return True + + def _conditional_remove(self, do_remove, filepath): + """ + Removes the message if do_remove is True + + @param do_remove: True if the message should be removed, False + otherwise + @type do_remove: bool + @param filepath: path to the mail + @type filepath: twisted.python.filepath.FilePath + """ + if do_remove: + # remove the original mail + try: + log.msg("Removing %s" % (filepath.path,)) + filepath.remove() + log.msg("Done removing") + except: + log.err() + + def _process_incoming_email(self, otherself, filepath, mask): + """ + Callback that processes incoming email + + @param otherself: Watch object for the current callback from + inotify + @type otherself: twisted.internet.inotify._Watch + @param filepath: Path of the file that changed + @type filepath: twisted.python.filepath.FilePath + @param mask: identifier for the type of change that triggered + this callback + @type mask: int + """ + if os.path.split(filepath.dirname())[-1] == "new": + log.msg("Processing new mail at %s" % (filepath.path,)) + with filepath.open("r") as f: + mail_data = f.read() + mail = message_from_string(mail_data) + owner = mail["To"] + if owner is None: # default to Delivered-To + owner = mail["Delivered-To"] + owner = owner.split("@")[0] + owner = owner.split("+")[0] + log.msg("Mail owner: %s" % (owner,)) + + log.msg("%s received a new mail" % (owner,)) + d = self._users_cdb.queryByLoginOrAlias(owner) + d.addCallbacks(self._get_pubkey, log.err) + d.addCallbacks(self._encrypt_message, log.err, (owner, mail_data)) + d.addCallbacks(self._export_message, log.err) + d.addCallbacks(self._conditional_remove, log.err, (filepath,)) + d.addErrback(log.err) - logger.info("Reading configuration from %s" % (config_file,)) - - config = ConfigParser.ConfigParser() - config.read(config_file) - - users_user = config.get("couchdb", "users_user") - users_password = config.get("couchdb", "users_password") - - mail_user = config.get("couchdb", "mail_user") - mail_password = config.get("couchdb", "mail_password") - - server = config.get("couchdb", "server") - port = config.get("couchdb", "port") - - wm = inotify.INotify(reactor) - wm.startReading() - - mask = inotify.IN_CREATE - - users_db = couchdbhelper.ConnectedCouchDB(server, - port=port, - dbName="users", - username=users_user, - password=users_password) - - mail_couch_url_prefix = "http://%s:%s@localhost:%s" % (mail_user, - mail_password, - port) - - incoming_partial = partial(_process_incoming_email, users_db, mail_couch_url_prefix) - for section in config.sections(): - if section in ("couchdb"): - continue - to_watch = config.get(section, "path") - recursive = config.getboolean(section, "recursive") - logger.debug("Watching %s --- Recursive: %s" % (to_watch, recursive)) - wm.watch(filepath.FilePath(to_watch), mask, callbacks=[incoming_partial], recursive=recursive) - - reactor.run() - -if __name__ == "__main__": - main() diff --git a/start_mx.py b/start_mx.py deleted file mode 100755 index bf3dddb..0000000 --- a/start_mx.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -# start_mx.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import argparse -import ConfigParser -import logging - -from functools import partial - -from leap.mx import couchdbhelper, mail_receiver -from leap.mx.alias_resolver import AliasResolverFactory -from leap.mx.check_recipient_access import CheckRecipientAccessFactory - -try: - from twisted.internet import reactor, inotify - from twisted.internet.endpoints import TCP4ServerEndpoint - from twisted.python import filepath -except ImportError, ie: - print "This software requires Twisted>=12.0.2, please see the README for" - print "help on using virtualenv and pip to obtain requirements." - - -if __name__ == "__main__": - epilog = "Copyright 2012 The LEAP Encryption Access Project" - parser = argparse.ArgumentParser(description="""LEAP MX""", - epilog=epilog) - parser.add_argument('-d', '--debug', action="store_true", - help="Launches the LEAP MX mail receiver with debug output") - parser.add_argument('-l', '--logfile', metavar="LOG FILE", nargs='?', - action="store", dest="log_file", - help="Writes the logs to the specified file") - parser.add_argument('-c', '--config', metavar="CONFIG FILE", nargs='?', - action="store", dest="config", - help="Where to look for the configuration file. " \ - "Default: mail_receiver.cfg") - - opts, _ = parser.parse_known_args() - - logger = logging.getLogger(name='leap') - - debug = opts.debug - config_file = opts.config - logfile = opts.log_file - - if debug: - level = logging.DEBUG - else: - level = logging.WARNING - - if config_file is None: - config_file = "mx.conf" - - logger.setLevel(level) - console = logging.StreamHandler() - console.setLevel(level) - formatter = logging.Formatter( - '%(asctime)s ' - '- %(name)s - %(levelname)s - %(message)s') - console.setFormatter(formatter) - logger.addHandler(console) - - if logfile is not None: - logger.debug('Setting logfile to %s ', logfile) - fileh = logging.FileHandler(logfile) - fileh.setLevel(logging.DEBUG) - fileh.setFormatter(formatter) - logger.addHandler(fileh) - - logger.info("~~~~~~~~~~~~~~~~~~~") - logger.info(" LEAP MX") - logger.info("~~~~~~~~~~~~~~~~~~~") - - logger.info("Reading configuration from %s" % (config_file,)) - - config = ConfigParser.ConfigParser() - config.read(config_file) - - user = config.get("couchdb", "user") - password = config.get("couchdb", "password") - - server = config.get("couchdb", "server") - port = config.get("couchdb", "port") - - cdb = couchdbhelper.ConnectedCouchDB(server, - port=port, - dbName="users", - username=user, - password=password) - - # Mail receiver - wm = inotify.INotify(reactor) - wm.startReading() - - mask = inotify.IN_CREATE - - mail_couch_url_prefix = "http://%s:%s@%s:%s" % (user, - password, - server, - port) - - incoming_partial = partial(mail_receiver._process_incoming_email, cdb, mail_couch_url_prefix) - for section in config.sections(): - if section in ("couchdb"): - continue - to_watch = config.get(section, "path") - recursive = config.getboolean(section, "recursive") - logger.debug("Watching %s --- Recursive: %s" % (to_watch, recursive)) - wm.watch(filepath.FilePath(to_watch), mask, callbacks=[incoming_partial], recursive=recursive) - - - # Alias map - alias_endpoint = TCP4ServerEndpoint(reactor, 4242) - alias_endpoint.listen(AliasResolverFactory(couchdb=cdb)) - - # Check recipient access - check_recipient = TCP4ServerEndpoint(reactor, 2244) - check_recipient.listen(CheckRecipientAccessFactory(couchdb=cdb)) - - reactor.run() -- cgit v1.2.3 From 406464efb2d2589a010e95bccfc6733a8e49c4f5 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Fri, 3 May 2013 08:17:16 -0300 Subject: Use the new naming for the soledad document json --- src/leap/mx/mail_receiver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 2494a21..49d4455 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -114,8 +114,8 @@ class MailReceiver(Service): if pubkey is None or len(pubkey) == 0: doc.content = { - "_encryption_scheme": EncryptionSchemes.NONE, - "_unencrypted_json": json.dumps(data) + "_enc_scheme": EncryptionSchemes.NONE, + "_enc_json": json.dumps(data) } return uuid, doc @@ -126,8 +126,8 @@ class MailReceiver(Service): openpgp_key = openpgp._safe_call(_ascii_to_openpgp_cb, pubkey) doc.content = { - "_encryption_scheme": EncryptionSchemes.PUBKEY, - "_encrypted_json": openpgp.encrypt_asym(json.dumps(data), openpgp_key) + "_enc_scheme": EncryptionSchemes.PUBKEY, + "_enc_json": openpgp.encrypt_asym(json.dumps(data), openpgp_key) } return uuid, doc -- cgit v1.2.3 From 549952c9acc0eb8dab750b8f43f1162f910f0fed Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Fri, 3 May 2013 17:05:53 -0300 Subject: Add setup script and init.d script Also, some pep8 fixes --- .gitignore | 6 ++++ MANIFEST.in | 1 + mx.tac | 3 +- pkg/__init__.py | 0 pkg/leap_mx | 50 ++++++++++++++++++++++++++++++ pkg/mx-requirements.pip | 4 --- pkg/requirements.pip | 12 ++++++++ pkg/utils/__init__.py | 0 pkg/utils/reqs.py | 72 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 50 ++++++++++++++++++++++-------- src/leap/__init__.py | 28 ++++------------- src/leap/mx/__init__.py | 2 -- src/leap/mx/alias_resolver.py | 3 +- src/leap/mx/couchdbhelper.py | 2 ++ src/leap/mx/mail_receiver.py | 16 +++++++--- 15 files changed, 202 insertions(+), 47 deletions(-) create mode 100644 MANIFEST.in create mode 100644 pkg/__init__.py create mode 100644 pkg/leap_mx delete mode 100644 pkg/mx-requirements.pip create mode 100644 pkg/requirements.pip create mode 100644 pkg/utils/__init__.py create mode 100644 pkg/utils/reqs.py diff --git a/.gitignore b/.gitignore index 0519df8..4c572d1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ develop-eggs .installed.cfg lib lib64 +MANIFEST +PKG-INFO # Installer logs pip-log.txt @@ -37,6 +39,10 @@ nosetests.xml # Don't upload private things *.private +# and vim's +*.swp +*.swo + # Ignore emacs temporary files *~ \#*\# diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..07f43b8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +data/* \ No newline at end of file diff --git a/mx.tac b/mx.tac index 30ede8f..cdebba7 100755 --- a/mx.tac +++ b/mx.tac @@ -1,6 +1,5 @@ -#!/usr/bin/env python # -*- encoding: utf-8 -*- -# start_mx.py +# mx.tac # Copyright (C) 2013 LEAP # # This program is free software: you can redistribute it and/or modify diff --git a/pkg/__init__.py b/pkg/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pkg/leap_mx b/pkg/leap_mx new file mode 100644 index 0000000..d2c4bc3 --- /dev/null +++ b/pkg/leap_mx @@ -0,0 +1,50 @@ +#!/bin/sh + +PATH=/sbin:/bin:/usr/sbin:/usr/bin + +pidfile=/var/run/leap_mx.pid \ +rundir=/var/lib/leap_mx/ \ +file=/etc/leap/mx \ +logfile=/var/log/leap_mx.log + +[ -r /etc/default/leap_mx ] && . /etc/default/leap_mx + +test -x /usr/bin/twistd || exit 0 +test -r $file || exit 0 +test -r /etc/leap/ || exit 0 + + +case "$1" in + start) + echo -n "Starting leap_mx: twistd" + start-stop-daemon --start --quiet --exec /usr/bin/twistd -- \ + --pidfile=$pidfile \ + --rundir=$rundir \ + --python=$file \ + --logfile=$logfile + echo "." + ;; + + stop) + echo -n "Stopping leap_mx: twistd" + start-stop-daemon --stop --quiet \ + --pidfile $pidfile + echo "." + ;; + + restart) + $0 stop + $0 start + ;; + + force-reload) + $0 restart + ;; + + *) + echo "Usage: /etc/init.d/leap_mx {start|stop|restart|force-reload}" >&2 + exit 1 + ;; +esac + +exit 0 \ No newline at end of file diff --git a/pkg/mx-requirements.pip b/pkg/mx-requirements.pip deleted file mode 100644 index a05ac98..0000000 --- a/pkg/mx-requirements.pip +++ /dev/null @@ -1,4 +0,0 @@ -Twisted -paisley>=0.3.1 -## xxx change me to whatever you name the package in pypi -python-gnupg>=0.3.0 diff --git a/pkg/requirements.pip b/pkg/requirements.pip new file mode 100644 index 0000000..dc39dca --- /dev/null +++ b/pkg/requirements.pip @@ -0,0 +1,12 @@ +Twisted>=12.0.2 +paisley>=0.3.1 +## XXX change me to whatever you name the package in pypi +python-gnupg>=0.3.0 +leap.common>=0.0.2-dev + +############### +# Development # +############### + +#leap.soledad # make this a dep as soon as it is installable from pypi !! +-e git://github.com/andrejb/soledad.git@develop#egg=leap.soledad diff --git a/pkg/utils/__init__.py b/pkg/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pkg/utils/reqs.py b/pkg/utils/reqs.py new file mode 100644 index 0000000..5e2324f --- /dev/null +++ b/pkg/utils/reqs.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# reqs.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Utils to help in the setup process +""" +import os +import re +import sys + + +def get_reqs_from_files(reqfiles): + """ + Returns the contents of the top requirement file listed as a + string list with the lines + + @param reqfiles: requirement files to parse + @type reqfiles: list of str + """ + for reqfile in reqfiles: + if os.path.isfile(reqfile): + return open(reqfile, 'r').read().split('\n') + + +def parse_requirements(reqfiles=['requirements.txt', + 'requirements.pip', + 'pkg/requirements.pip']): + """ + Parses the requirement files provided. + + @param reqfiles: requirement files to parse + @type reqfiles: list of str + """ + + requirements = [] + for line in get_reqs_from_files(reqfiles): + # -e git://foo.bar/baz/master#egg=foobar + if re.match(r'\s*-e\s+', line): + pass + # do not try to do anything with externals on vcs + #requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', r'\1', + #line)) + # http://foo.bar/baz/foobar/zipball/master#egg=foobar + elif re.match(r'\s*https?:', line): + requirements.append(re.sub(r'\s*https?:.*#egg=(.*)$', r'\1', + line)) + # -f lines are for index locations, and don't get used here + elif re.match(r'\s*-f\s+', line): + pass + + # argparse is part of the standard library starting with 2.7 + # adding it to the requirements list screws distro installs + elif line == 'argparse' and sys.version_info >= (2, 7): + pass + else: + if line != '': + requirements.append(line) + + return requirements diff --git a/setup.py b/setup.py index efceb4c..c511a10 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,48 @@ # -*- coding: utf-8 -*- # setup.py -# Copyleft (C) 2013 LEAP +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . """ setup file for leap.mx """ from setuptools import setup, find_packages -requirements = [ - "twisted", - #... -] +from pkg.utils.reqs import parse_requirements -# XXX add classifiers, docs +trove_classifiers = [ + 'Development Status :: 3 - Alpha', + 'Environment :: No Input/Output (Daemon)', + 'Framework :: Twisted', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Affero General Public License v3' + 'or later (AGPLv3+)', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Topic :: Communications :: Email', + 'Topic :: Security :: Cryptography', +] setup( name='leap.mx', - version='0.0.1', - url='https://leap.se/', - license='', - author="Isis Agora Lovecruft", - author_email=" 0x2cdb8b35", + version="0.2.0", + url="http://github.com/leapcode/leap_mx", + license='AGPLv3+', + author='The LEAP Encryption Access Project', + author_email='info@leap.se', description=("An asynchronous, transparently-encrypting remailer " "for the LEAP platform"), long_description=( @@ -30,5 +53,8 @@ setup( package_dir={'': 'src'}, packages=find_packages('src'), #test_suite='leap.mx.tests', - install_requires=requirements, + install_requires=parse_requirements(), + classifiers=trove_classifiers, + data_files = [("/etc/leap/", ["mx.tac"]), + ("/etc/init.d/", ["pkg/leap_mx"])] ) diff --git a/src/leap/__init__.py b/src/leap/__init__.py index ff2d8a1..f48ad10 100644 --- a/src/leap/__init__.py +++ b/src/leap/__init__.py @@ -1,22 +1,6 @@ -# -*- encoding: utf-8 -*- -# __init__.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -Module intialization file for leap. -""" - -__all__ = ['mx'] +# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages +try: + __import__('pkg_resources').declare_namespace(__name__) +except ImportError: + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) diff --git a/src/leap/mx/__init__.py b/src/leap/mx/__init__.py index efb28ae..e0aebd8 100644 --- a/src/leap/mx/__init__.py +++ b/src/leap/mx/__init__.py @@ -14,10 +14,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - """ Module initialization file for leap.mx . """ -from leap.mx.util import version __all__ = ['alias_resolver', 'couchdbhelper'] diff --git a/src/leap/mx/alias_resolver.py b/src/leap/mx/alias_resolver.py index 68c6212..2074ee5 100644 --- a/src/leap/mx/alias_resolver.py +++ b/src/leap/mx/alias_resolver.py @@ -36,7 +36,8 @@ except ImportError: class AliasResolverFactory(postfix.PostfixTCPMapDeferringDictServerFactory): def __init__(self, couchdb, *args, **kwargs): - postfix.PostfixTCPMapDeferringDictServerFactory.__init__(self, *args, **kwargs) + postfix.PostfixTCPMapDeferringDictServerFactory.__init__( + self, *args, **kwargs) self._cdb = couchdb def _to_str(self, result): diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index 7c4c8ce..02ef088 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -157,6 +157,7 @@ if __name__ == "__main__": password="") d = cdb.queryByLoginOrAlias("test1") + @d.addCallback def right(result): print "Should be an actual uuid:", result @@ -164,6 +165,7 @@ if __name__ == "__main__": print cdb.getPubKey(result) d2 = cdb.queryByLoginOrAlias("asdjaoisdjoiqwjeoi") + @d2.addCallback def wrong(result): print "Should be None:", result diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 49d4455..2c04863 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -72,7 +72,9 @@ class MailReceiver(Service): for directory, recursive in self._directories: log.msg("Watching %s --- Recursive: %s" % (directory, recursive)) - wm.watch(filepath.FilePath(directory), mask, callbacks=[self._process_incoming_email], recursive=recursive) + wm.watch(filepath.FilePath(directory), mask, + callbacks=[self._process_incoming_email], + recursive=recursive) def _get_pubkey(self, uuid): """ @@ -127,7 +129,8 @@ class MailReceiver(Service): doc.content = { "_enc_scheme": EncryptionSchemes.PUBKEY, - "_enc_json": openpgp.encrypt_asym(json.dumps(data), openpgp_key) + "_enc_json": openpgp.encrypt_asym(json.dumps(data), + openpgp_key) } return uuid, doc @@ -198,6 +201,9 @@ class MailReceiver(Service): owner = mail["To"] if owner is None: # default to Delivered-To owner = mail["Delivered-To"] + if owner is None: + log.err("Malformed mail, neither To: nor " + "Delivered-To: field") owner = owner.split("@")[0] owner = owner.split("+")[0] log.msg("Mail owner: %s" % (owner,)) @@ -205,8 +211,10 @@ class MailReceiver(Service): log.msg("%s received a new mail" % (owner,)) d = self._users_cdb.queryByLoginOrAlias(owner) d.addCallbacks(self._get_pubkey, log.err) - d.addCallbacks(self._encrypt_message, log.err, (owner, mail_data)) + d.addCallbacks(self._encrypt_message, log.err, + (owner, mail_data)) d.addCallbacks(self._export_message, log.err) - d.addCallbacks(self._conditional_remove, log.err, (filepath,)) + d.addCallbacks(self._conditional_remove, log.err, + (filepath,)) d.addErrback(log.err) -- cgit v1.2.3 From d926aef5812fb648c278c289b08f93e8d7b02948 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Mon, 6 May 2013 11:10:00 -0300 Subject: Improve leap_mx init script --- pkg/leap_mx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/leap_mx b/pkg/leap_mx index d2c4bc3..d18d44f 100644 --- a/pkg/leap_mx +++ b/pkg/leap_mx @@ -4,7 +4,7 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin pidfile=/var/run/leap_mx.pid \ rundir=/var/lib/leap_mx/ \ -file=/etc/leap/mx \ +file=/etc/leap/mx.tac \ logfile=/var/log/leap_mx.log [ -r /etc/default/leap_mx ] && . /etc/default/leap_mx @@ -17,7 +17,8 @@ test -r /etc/leap/ || exit 0 case "$1" in start) echo -n "Starting leap_mx: twistd" - start-stop-daemon --start --quiet --exec /usr/bin/twistd -- \ + HOME="/var/lib/leap_mx/" \ + start-stop-daemon --start --quiet --exec /usr/bin/twistd -- \ --pidfile=$pidfile \ --rundir=$rundir \ --python=$file \ @@ -47,4 +48,4 @@ case "$1" in ;; esac -exit 0 \ No newline at end of file +exit 0 -- cgit v1.2.3 From b15d733c3b3022c784d74abc4320ae1158364c3a Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 7 May 2013 13:40:16 -0300 Subject: Update readme to point to leapcode's repo --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5b4ccf3..28c483d 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ is provided. It does the following: virtualenvwrapper in the bootstrap virtualenv. 4. Obtain a copy of leap_mx with git clone. 5. Use ```mkvirtualenv``` included in the virtualenvwrapper inside the - bootstrap virtualenv to install a project virtualenv for leap_mx. + bootstrap virtualenv to install a project virtualenv for leap_mx. To use the bootstrap script, do: ~~~ @@ -43,7 +43,7 @@ To install python, virtualenv, and get started, do: ~~~ $ sudo apt-get install python2.7 python-dev python-virtualenv virtualenvwrapper -$ git clone https://github.com/isislovecruft/leap_mx.git leap_mx +$ git clone https://github.com/leapcode/leap_mx.git leap_mx $ export WORKON_LEAPMX=${PWD}/leap_mx $ source /usr/local/bin/virtualenvwrapper.sh $ mkvirtualenv -a $WORKON_LEAPMX -r ${WORKON_LEAPMX}/pkg/mx-requirements.pip \ @@ -54,10 +54,10 @@ $ mkvirtualenv -a $WORKON_LEAPMX -r ${WORKON_LEAPMX}/pkg/mx-requirements.pip \ To get started quickly, without virtualenv, do: ~~~ $ sudo apt-get install python git -$ git clone https://github.com/isislovecruft/leap_mx.git +$ git clone https://github.com/leapcode/leap_mx.git # pip install -r ./leap_mx/pkg/mx-requirements.pip ~~~ -Although, **it is advised** to install inside a python virtualenv. +Although, **it is advised** to install inside a python virtualenv. ## [running](#running) ## ========================= @@ -73,7 +73,7 @@ $ ./start_mx.py --help ========================= Please see the HACKING and DESIGN docs. -Our bugtracker is [here](https://leap.se/code/projects/eip_server/issue/new). +Our bugtracker is [here](https://leap.se/code/projects/eip_server/issue/new). Please use that for bug reports and feature requests instead of github's tracker. We're using github for code commenting and review between -- cgit v1.2.3 From ef97f1f6daa07fe1915cb459c6cf7a8dd5cc9578 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 7 May 2013 13:45:40 -0300 Subject: Fix classifier --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c511a10..5353bbc 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ trove_classifiers = [ 'License :: OSI Approved :: GNU Affero General Public License v3' 'or later (AGPLv3+)', 'Natural Language :: English', - 'Operating System :: OS Independent', + 'Operating System :: POSIX :: Linux', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Topic :: Communications :: Email', -- cgit v1.2.3 From 9c4679c0bfa398c1c3adbf151411d61a4df9852c Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 7 May 2013 13:46:57 -0300 Subject: Move sample-config to doc and update sample --- doc/sample-config/mx.conf | 15 +++++++++++++++ sample-config/mx.conf | 9 --------- 2 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 doc/sample-config/mx.conf delete mode 100644 sample-config/mx.conf diff --git a/doc/sample-config/mx.conf b/doc/sample-config/mx.conf new file mode 100644 index 0000000..48d4222 --- /dev/null +++ b/doc/sample-config/mx.conf @@ -0,0 +1,15 @@ +[mail1] +path=/home/blabla/Maildir/ +recursive=True + +[couchdb] +user=someuser +password=somepass +server=localhost +port=5984 + +[alias map] +port=2222 + +[check recipient] +port=2233 diff --git a/sample-config/mx.conf b/sample-config/mx.conf deleted file mode 100644 index 2036706..0000000 --- a/sample-config/mx.conf +++ /dev/null @@ -1,9 +0,0 @@ -[mail1] -path=/home/blabla/Maildir/ -recursive=True - -[couchdb] -user=someuser -password=somepass -server=localhost -port=6666 -- cgit v1.2.3 From fbed553ede006f02ed2c65e0744a2fb3d7817582 Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 7 May 2013 13:48:07 -0300 Subject: Place mx.tac in /usr/local/bin/ --- pkg/leap_mx | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/leap_mx b/pkg/leap_mx index d18d44f..0b50426 100644 --- a/pkg/leap_mx +++ b/pkg/leap_mx @@ -4,7 +4,7 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin pidfile=/var/run/leap_mx.pid \ rundir=/var/lib/leap_mx/ \ -file=/etc/leap/mx.tac \ +file=/usr/local/bin/mx.tac \ logfile=/var/log/leap_mx.log [ -r /etc/default/leap_mx ] && . /etc/default/leap_mx diff --git a/setup.py b/setup.py index 5353bbc..c2ea309 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,6 @@ setup( #test_suite='leap.mx.tests', install_requires=parse_requirements(), classifiers=trove_classifiers, - data_files = [("/etc/leap/", ["mx.tac"]), + data_files = [("/usr/local/bin/", ["mx.tac"]), ("/etc/init.d/", ["pkg/leap_mx"])] ) -- cgit v1.2.3 From de214ccc07827c5ebad70588fa0096392f49f9ba Mon Sep 17 00:00:00 2001 From: Tomas Touceda Date: Tue, 7 May 2013 13:56:18 -0300 Subject: Update README some more and bump leap.common --- README.md | 8 ++++---- pkg/requirements.pip | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 28c483d..c467496 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ $ sudo apt-get install python2.7 python-dev python-virtualenv virtualenvwrapper $ git clone https://github.com/leapcode/leap_mx.git leap_mx $ export WORKON_LEAPMX=${PWD}/leap_mx $ source /usr/local/bin/virtualenvwrapper.sh -$ mkvirtualenv -a $WORKON_LEAPMX -r ${WORKON_LEAPMX}/pkg/mx-requirements.pip \ +$ mkvirtualenv -a $WORKON_LEAPMX -r ${WORKON_LEAPMX}/pkg/requirements.pip \ --no-site-packages --setuptools --unzip-setuptools leap_mx ~~~ @@ -55,7 +55,7 @@ To get started quickly, without virtualenv, do: ~~~ $ sudo apt-get install python git $ git clone https://github.com/leapcode/leap_mx.git -# pip install -r ./leap_mx/pkg/mx-requirements.pip +# pip install -r ./leap_mx/pkg/requirements.pip ~~~ Although, **it is advised** to install inside a python virtualenv. @@ -66,14 +66,14 @@ To get running, clone this repo, and (assuming you've already set up your virtualenv and obtained all the requirements) do: ~~~ -$ ./start_mx.py --help +$ twistd -ny mx.tac ~~~ ## [hacking](#hacking) ## ========================= Please see the HACKING and DESIGN docs. -Our bugtracker is [here](https://leap.se/code/projects/eip_server/issue/new). +Our bugtracker is [here](https://leap.se/code/projects/eip/issue/new). Please use that for bug reports and feature requests instead of github's tracker. We're using github for code commenting and review between diff --git a/pkg/requirements.pip b/pkg/requirements.pip index dc39dca..c09ce53 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -2,7 +2,7 @@ Twisted>=12.0.2 paisley>=0.3.1 ## XXX change me to whatever you name the package in pypi python-gnupg>=0.3.0 -leap.common>=0.0.2-dev +leap.common>=0.2.3-dev ############### # Development # -- cgit v1.2.3