From bc2fea9affe64ffc157cb70eec68bfd776dad76f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 15 Oct 2013 22:46:13 -0300 Subject: pep8 and sphinx format --- src/leap/mx/couchdbhelper.py | 44 ++++++++++++------------ src/leap/mx/mail_receiver.py | 81 ++++++++++++++++++++++---------------------- 2 files changed, 62 insertions(+), 63 deletions(-) diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index 147e6f9..b500f17 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -49,16 +49,16 @@ class ConnectedCouchDB(client.CouchDB): """ Connect to a CouchDB instance. - @param host: A hostname string for the CouchDB server. - @type host: str - @param port: The port of the CouchDB server. - @type port: int - @param dbName: (optional) The default database to bind queries to. - @type dbName: str - @param username: (optional) The username for authorization. - @type username: str - @param str password: (optional) The password for authorization. - @type password: str + :param host: A hostname string for the CouchDB server. + :type host: str + :param port: The port of the CouchDB server. + :type port: int + :param dbName: (optional) The default database to bind queries to. + :type dbName: str + :param username: (optional) The username for authorization. + :type username: str + :param str password: (optional) The password for authorization. + :type password: str """ client.CouchDB.__init__(self, host, @@ -78,8 +78,8 @@ class ConnectedCouchDB(client.CouchDB): """ Callback for listDB that prints the available databases - @param data: response from the listDB command - @type data: array + :param data: response from the listDB command + :type data: array """ log.msg("Available databases:") for database in data: @@ -101,10 +101,10 @@ class ConnectedCouchDB(client.CouchDB): """ Check to see if a particular email or alias exists. - @param alias: A string representing the email or alias to check. - @type alias: str - @return: a deferred for this query - @rtype twisted.defer.Deferred + :param alias: A string representing the email or alias to check. + :type alias: str + :return: a deferred for this query + :rtype twisted.defer.Deferred """ assert isinstance(address, (str, unicode)), "Email or alias queries must be string" @@ -124,12 +124,12 @@ class ConnectedCouchDB(client.CouchDB): """ Parses the result of the by_address query and gets the uuid - @param address: alias looked up - @type address: string - @param result: result dictionary - @type result: dict - @return: The uuid for alias if available - @rtype: str + :param address: alias looked up + :type address: string + :param result: result dictionary + :type result: dict + :return: The uuid for alias if available + :rtype: str """ for row in result["rows"]: if row["key"] == address: diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 5875034..e263604 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -15,11 +15,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - """ MailReceiver service definition """ - import os import uuid as pyuuid @@ -54,14 +52,14 @@ class MailReceiver(Service): """ Constructor - @param mail_couch_url: URL prefix for the couchdb where mail + :param mail_couch_url: URL prefix for the couchdb where mail should be stored - @type mail_couch_url: str - @param users_cdb: CouchDB instance from where to get the uuid + :type mail_couch_url: str + :param users_cdb: CouchDB instance from where to get the uuid and pubkey for a user - @type users_cdb: ConnectedCouchDB - @param directories: list of directories to monitor - @type directories: list of tuples (path: str, recursive: bool) + :type users_cdb: ConnectedCouchDB + :param directories: list of directories to monitor + :type directories: list of tuples (path: str, recursive: bool) """ # Service doesn't define an __init__ self._mail_couch_url = mail_couch_url @@ -105,16 +103,17 @@ class MailReceiver(Service): the message to that public key. The address is needed in order to build the OpenPGPKey object. - @param uuid_pubkey: tuple that holds the uuid and the public - key as it is returned by the previous call in the chain - @type uuid_pubkey: tuple (str, str) - @param address: mail address for this message - @type address: str - @param message: message contents - @type message: str - - @return: uuid, doc to sync with Soledad - @rtype: tuple(str, SoledadDocument) + :param uuid_pubkey: tuple that holds the uuid and the public + key as it is returned by the previous call in the + chain + :type uuid_pubkey: tuple (str, str) + :param address: mail address for this message + :type address: str + :param message: message contents + :type message: str + + :return: uuid, doc to sync with Soledad + :rtype: tuple(str, SoledadDocument) """ uuid, pubkey = uuid_pubkey log.msg("Encrypting message to %s's pubkey" % (uuid,)) @@ -153,14 +152,14 @@ class MailReceiver(Service): """ Given a UUID and a SoledadDocument, it saves it directly in the couchdb that serves as a backend for Soledad, in a db - accessible to the recipient of the mail + accessible to the recipient of the mail. - @param uuid_doc: tuple that holds the UUID and SoledadDocument - @type uuid_doc: tuple(str, SoledadDocument) + :param uuid_doc: tuple that holds the UUID and SoledadDocument + :type uuid_doc: tuple(str, SoledadDocument) - @return: True if it's ok to remove the message, False - otherwise - @rtype: bool + :return: True if it's ok to remove the message, False + otherwise + :rtype: bool """ uuid, doc = uuid_doc log.msg("Exporting message for %s" % (uuid,)) @@ -177,13 +176,13 @@ class MailReceiver(Service): def _conditional_remove(self, do_remove, filepath): """ - Removes the message if do_remove is True + Removes the message if do_remove is True. - @param do_remove: True if the message should be removed, False - otherwise - @type do_remove: bool - @param filepath: path to the mail - @type filepath: twisted.python.filepath.FilePath + :param do_remove: True if the message should be removed, False + otherwise + :type do_remove: bool + :param filepath: path to the mail + :type filepath: twisted.python.filepath.FilePath """ if do_remove: # remove the original mail @@ -196,18 +195,18 @@ class MailReceiver(Service): def _process_incoming_email(self, otherself, filepath, mask): """ - Callback that processes incoming email - - @param otherself: Watch object for the current callback from - inotify - @type otherself: twisted.internet.inotify._Watch - @param filepath: Path of the file that changed - @type filepath: twisted.python.filepath.FilePath - @param mask: identifier for the type of change that triggered - this callback - @type mask: int + Callback that processes incoming email. + + :param otherself: Watch object for the current callback from + inotify. + :type otherself: twisted.internet.inotify._Watch + :param filepath: Path of the file that changed + :type filepath: twisted.python.filepath.FilePath + :param mask: identifier for the type of change that triggered + this callback + :type mask: int """ - if os.path.split(filepath.dirname())[-1] == "new": + if os.path.split(filepath.dirname())[-1] == "new": log.msg("Processing new mail at %s" % (filepath.path,)) with filepath.open("r") as f: mail_data = f.read() -- cgit v1.2.3 From 56bd5becca970c02c19d1b4b9a26cbbf5cb62d17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Thu, 24 Oct 2013 13:41:49 -0300 Subject: Support any encoding in the emails --- pkg/requirements.pip | 2 ++ src/leap/mx/mail_receiver.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pkg/requirements.pip b/pkg/requirements.pip index d5db275..06e3b85 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -12,3 +12,5 @@ couchdb python-gnupg>=0.3.0 leap.soledad.common>=0.3.0 leap.keymanager>=0.2.0 + +cchardet # we fallback to chardet if this is not available, but it's preferred \ No newline at end of file diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index e263604..c7652fc 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -15,14 +15,22 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + """ MailReceiver service definition """ + import os import uuid as pyuuid import json import email.utils +import socket + +try: + import cchardet as chardet +except ImportError: + import chardet from email import message_from_string @@ -121,13 +129,17 @@ class MailReceiver(Service): doc = SoledadDocument(doc_id=str(pyuuid.uuid4())) + result = chardet.detect(message) + + encoding = result["encoding"] + data = {'incoming': True, 'content': message} if pubkey is None or len(pubkey) == 0: doc.content = { self.INCOMING_KEY: True, ENC_SCHEME_KEY: EncryptionSchemes.NONE, - ENC_JSON_KEY: json.dumps(data) + ENC_JSON_KEY: json.dumps(data, encoding=encoding) } return uuid, doc @@ -141,7 +153,7 @@ class MailReceiver(Service): self.INCOMING_KEY: True, ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, ENC_JSON_KEY: str(gpg.encrypt( - json.dumps(data), + json.dumps(data, encoding=encoding), openpgp_key.fingerprint, symmetric=False)) } -- cgit v1.2.3 From adea30df46100e2c66c3691ccf6242d317abb4ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Thu, 24 Oct 2013 13:42:42 -0300 Subject: Make the mail received callback more robust and add support for ml --- src/leap/mx/couchdbhelper.py | 30 ++++++++--- src/leap/mx/mail_receiver.py | 115 ++++++++++++++++++++++++++----------------- 2 files changed, 94 insertions(+), 51 deletions(-) diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py index b500f17..41604ba 100644 --- a/src/leap/mx/couchdbhelper.py +++ b/src/leap/mx/couchdbhelper.py @@ -29,7 +29,6 @@ except ImportError: print "for instructions on getting required dependencies." try: - from twisted.internet import defer from twisted.python import log except ImportError: print "This software requires Twisted. Please see the README file" @@ -140,20 +139,39 @@ class ConnectedCouchDB(client.CouchDB): return uuid return None - def getPubKey(self, address): + def getPubKey(self, uuid): + """ + Returns a deferred that will return the pubkey for the uuid provided + + :param uuid: uuid for the user to query + :type uuid: str + + :rtype: Deferred + """ d = self.openView(docId="Identity", viewId="pgp_key_by_email/", - key=address, + user_id=uuid, reduce=False, include_docs=True) - d.addCallbacks(partial(self._get_pgp_key, address), log.err) + d.addCallbacks(partial(self._get_pgp_key, uuid), log.err) return d - def _get_pgp_key(self, address, result): + def _get_pgp_key(self, uuid, result): + """ + Callback used to filter the correct pubkey from the result of + the query to the couchdb + + :param uuid: uuid for the user that was queried + :type uuid: str + :param result: result dictionary for the db query + :type result: dict + + :rtype: str or None + """ for row in result["rows"]: - if row["key"] == address: + if row["doc"]["user_id"] == uuid: return row["value"] return None diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index c7652fc..bae489f 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -36,7 +36,6 @@ from email import message_from_string from twisted.application.service import Service from twisted.internet import inotify -from twisted.internet.defer import DeferredList from twisted.python import filepath, log from leap.soledad.common.document import SoledadDocument @@ -73,6 +72,7 @@ class MailReceiver(Service): self._mail_couch_url = mail_couch_url self._users_cdb = users_cdb self._directories = directories + self._domain = socket.gethostbyaddr(socket.gethostname())[0] def startService(self): """ @@ -90,22 +90,7 @@ class MailReceiver(Service): callbacks=[self._process_incoming_email], recursive=recursive) - def _gather_uuid_pubkey(self, results): - if len(results) < 2: - return None, None - - # DeferredList results are structured like this: - # [ (succeeded, pubkey), (succeeded, uuid) ] - # succeeded is a bool value that specifies if the - # corresponding callback succeeded - pubkey_res, uuid_res = results - - pubkey = pubkey_res[1] if pubkey_res[0] else None - uuid = uuid_res[1] if uuid_res[0] else None - - return uuid, pubkey - - def _encrypt_message(self, uuid_pubkey, address, message): + def _encrypt_message(self, pubkey, uuid, address, message): """ Given a UUID, a public key, address and a message, it encrypts the message to that public key. @@ -120,10 +105,13 @@ class MailReceiver(Service): :param message: message contents :type message: str - :return: uuid, doc to sync with Soledad + :return: uuid, doc to sync with Soledad or None, None if + something went wrong. :rtype: tuple(str, SoledadDocument) """ - uuid, pubkey = uuid_pubkey + if uuid is None or pubkey is None or len(pubkey) == 0: + return None, None + log.msg("Encrypting message to %s's pubkey" % (uuid,)) log.msg("Pubkey: %s" % (pubkey,)) @@ -174,6 +162,9 @@ class MailReceiver(Service): :rtype: bool """ uuid, doc = uuid_doc + if uuid is None or doc is None: + return False + log.msg("Exporting message for %s" % (uuid,)) if uuid is None: @@ -202,9 +193,41 @@ class MailReceiver(Service): log.msg("Removing %s" % (filepath.path,)) filepath.remove() log.msg("Done removing") - except: + except Exception: log.err() + def _get_owner(self, mail): + """ + Given an email, returns the owner (who's delivered to) and the + uuid of the owner. In case of a mailing list mail, the owner + will end up being the mailing list address, but the owner's + uuid will be correct as long as the user exists in the system. + + :param mail: mail to analyze + :type mail: email.message.Message + + :returns: a tuple of (owner, uuid) + :rtype: tuple(str or None, str or None) + """ + owner = mail["To"] + uuid = None + + delivereds = mail.get_all("Delivered-To") + for to in delivereds: + name, addr = email.utils.parseaddr(to) + parts = addr.split("@") + if len(parts) > 1 and parts[1] == self._domain: + uuid = parts[0] + break + + if owner is None: # default to Delivered-To + owner = mail["Delivered-To"] + if owner is None: + log.err("Malformed mail, neither To: nor " + "Delivered-To: field") + + return owner, uuid + def _process_incoming_email(self, otherself, filepath, mask): """ Callback that processes incoming email. @@ -218,28 +241,30 @@ class MailReceiver(Service): this callback :type mask: int """ - if os.path.split(filepath.dirname())[-1] == "new": - log.msg("Processing new mail at %s" % (filepath.path,)) - with filepath.open("r") as f: - mail_data = f.read() - mail = message_from_string(mail_data) - owner = mail["To"] - if owner is None: # default to Delivered-To - owner = mail["Delivered-To"] - if owner is None: - log.err("Malformed mail, neither To: nor " - "Delivered-To: field") - log.msg("Mail owner: %s" % (owner,)) - - owner = email.utils.parseaddr(owner)[1] - log.msg("%s received a new mail" % (owner,)) - dpubk = self._users_cdb.getPubKey(owner) - duuid = self._users_cdb.queryByAddress(owner) - d = DeferredList([dpubk, duuid]) - d.addCallbacks(self._gather_uuid_pubkey, log.err) - d.addCallbacks(self._encrypt_message, log.err, - (owner, mail_data)) - d.addCallbacks(self._export_message, log.err) - d.addCallbacks(self._conditional_remove, log.err, - (filepath,)) - d.addErrback(log.err) + try: + if os.path.split(filepath.dirname())[-1] == "new": + log.msg("Processing new mail at %s" % (filepath.path,)) + with filepath.open("r") as f: + mail_data = f.read() + mail = message_from_string(mail_data) + owner, uuid = self._get_owner(mail) + if owner is None: + # This shouldn't happen, may be a bug in + # postfix? But we don't want to drop mail just + # because of a bug. Skipping this one... + return + log.msg("Mail owner: %s %s" % (owner, uuid)) + + if uuid is None: + log.msg("BUG: There was no uuid!") + return + + d = self._users_cdb.getPubKey(uuid) + d.addCallbacks(self._encrypt_message, log.err, + (uuid, owner, mail_data)) + d.addCallbacks(self._export_message, log.err) + d.addCallbacks(self._conditional_remove, log.err, + (filepath,)) + d.addErrback(log.err) + except Exception: + log.err() -- cgit v1.2.3 From a2c88d23c5b34926a3e9f9efbf005b2b4806094d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Thu, 24 Oct 2013 13:46:02 -0300 Subject: Add changes file --- changes/bug_support_more_than_utf8 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/bug_support_more_than_utf8 diff --git a/changes/bug_support_more_than_utf8 b/changes/bug_support_more_than_utf8 new file mode 100644 index 0000000..2e467aa --- /dev/null +++ b/changes/bug_support_more_than_utf8 @@ -0,0 +1,5 @@ + o Support more than utf8 encodings for emails. + o Add support for receiving mailing list mails. + o Use the uuid that alias_resolver returned and postfix added to the + mail headers, which improves performance. + o Look for public keys based on uuid instead of mail address. \ No newline at end of file -- cgit v1.2.3 From cd48c66406c39ca6dd6bdc6ba7c2be0df623e6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Tue, 29 Oct 2013 16:28:42 -0300 Subject: Fix return codes for check recipient --- src/leap/mx/check_recipient_access.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py index 0520c7c..b80ccfd 100644 --- a/src/leap/mx/check_recipient_access.py +++ b/src/leap/mx/check_recipient_access.py @@ -29,11 +29,13 @@ from leap.mx.alias_resolver import AliasResolverFactory class LEAPPostFixTCPMapserverAccess(postfix.PostfixTCPMapServer): def _cbGot(self, value): + # For more info, see: + # http://www.postfix.org/tcp_table.5.html + # http://www.postfix.org/access.5.html if value is None: - self.sendCode(500, postfix.quote("NOT FOUND SORRY")) + self.sendCode(500, postfix.quote("REJECT")) else: - # We do not send the value in this case - self.sendCode(200) + self.sendCode(200, postfix.quote("OK")) class CheckRecipientAccessFactory(AliasResolverFactory): -- cgit v1.2.3 From eecd4f4b1cfff41944d9fb002a56b8675fa05298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Tue, 29 Oct 2013 16:30:03 -0300 Subject: Add changes file --- changes/bug_fix_ret_codes | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/bug_fix_ret_codes diff --git a/changes/bug_fix_ret_codes b/changes/bug_fix_ret_codes new file mode 100644 index 0000000..54778db --- /dev/null +++ b/changes/bug_fix_ret_codes @@ -0,0 +1 @@ + o Fix return codes for check recipient access. Fixes #3356. \ No newline at end of file -- cgit v1.2.3 From c2e61c4d33aa73a31a2065fcd381af4250191f54 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 30 Oct 2013 12:03:39 -0200 Subject: add freeze_debianver --- setup.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3de66cf..6fec416 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ setup file for leap.mx """ import os +import re from setuptools import setup, find_packages import versioneer @@ -43,6 +44,68 @@ trove_classifiers = [ 'Topic :: Security :: Cryptography', ] +DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mx/' + 'archive/%s.tar.gz') +_versions = versioneer.get_versions() +VERSION = _versions['version'] +VERSION_FULL = _versions['full'] +DOWNLOAD_URL = "" + +# get the short version for the download url +_version_short = re.findall('\d+\.\d+\.\d+', VERSION) +if len(_version_short) > 0: + VERSION_SHORT = _version_short[0] + DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT + +cmdclass = versioneer.get_cmdclass() + + +from setuptools import Command + + +class freeze_debianver(Command): + """ + Freezes the version in a debian branch. + To be used after merging the development branch onto the debian one. + """ + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + proceed = str(raw_input( + "This will overwrite the file _version.py. Continue? [y/N] ")) + if proceed != "y": + print("He. You scared. Aborting.") + return + template = r""" +# This file was generated by the `freeze_debianver` command in setup.py +# Using 'versioneer.py' (0.7+) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +version_version = '{version}' +version_full = '{version_full}' +""" + templatefun = r""" + +def get_versions(default={}, verbose=False): + return {'version': version_version, 'full': version_full} +""" + subst_template = template.format( + version=VERSION_SHORT, + version_full=VERSION_FULL) + templatefun + with open(versioneer.versionfile_source, 'w') as f: + f.write(subst_template) + + +cmdclass["freeze_debianver"] = freeze_debianver + if os.environ.get("VIRTUAL_ENV", None): data_files = None else: @@ -54,12 +117,15 @@ else: ("/etc/init.d/", ["pkg/leap_mx"])] setup( name='leap.mx', - version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), + version=VERSION, + cmdclass=cmdclass, url="http://github.com/leapcode/leap_mx", + download_url=DOWNLOAD_URL, license='AGPLv3+', author='The LEAP Encryption Access Project', author_email='info@leap.se', + maintainer='Kali Kaneko', + maintainer_email='kali@leap.se', description=("An asynchronous, transparently-encrypting remailer " "for the LEAP platform"), long_description=( -- cgit v1.2.3 From 081642b69751cea8992cf03b06d8956684c2b126 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 30 Oct 2013 16:21:20 -0200 Subject: update mx requirements --- pkg/requirements.pip | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/requirements.pip b/pkg/requirements.pip index 06e3b85..fa5548b 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -8,9 +8,7 @@ paisley>=0.3.1 # for the time being. couchdb -## XXX change me to whatever you name the package in pypi -python-gnupg>=0.3.0 leap.soledad.common>=0.3.0 -leap.keymanager>=0.2.0 +leap.keymanager>=0.3.4 -cchardet # we fallback to chardet if this is not available, but it's preferred \ No newline at end of file +cchardet # we fallback to chardet if this is not available, but it's preferred -- cgit v1.2.3 From 527d941a0b87ca23d07c003dbe87dddf37c36b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Thu, 31 Oct 2013 07:57:25 -0300 Subject: Pick correct recipient and encoding --- src/leap/mx/mail_receiver.py | 51 ++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index bae489f..36cfe0b 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -38,6 +38,7 @@ from twisted.application.service import Service from twisted.internet import inotify from twisted.python import filepath, log +from leap.common.mail import get_email_charset from leap.soledad.common.document import SoledadDocument from leap.soledad.common.crypto import ( EncryptionSchemes, @@ -90,18 +91,16 @@ class MailReceiver(Service): callbacks=[self._process_incoming_email], recursive=recursive) - def _encrypt_message(self, pubkey, uuid, address, message): + def _encrypt_message(self, pubkey, uuid, message): """ - Given a UUID, a public key, address and a message, it encrypts - the message to that public key. + Given a UUID, a public key and a message, it encrypts the + message to that public key. The address is needed in order to build the OpenPGPKey object. :param uuid_pubkey: tuple that holds the uuid and the public key as it is returned by the previous call in the chain :type uuid_pubkey: tuple (str, str) - :param address: mail address for this message - :type address: str :param message: message contents :type message: str @@ -117,9 +116,10 @@ class MailReceiver(Service): doc = SoledadDocument(doc_id=str(pyuuid.uuid4())) - result = chardet.detect(message) - - encoding = result["encoding"] + encoding = get_email_charset(message, default=None) + if encoding is None: + result = chardet.detect(message) + encoding = result["encoding"] data = {'incoming': True, 'content': message} @@ -135,7 +135,9 @@ class MailReceiver(Service): with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg: gpg.import_keys(pubkey) key = gpg.list_keys().pop() - openpgp_key = openpgp._build_key_from_gpg(address, key, pubkey) + # We don't care about the actual address, so we use a + # dummy one, we just care about the import of the pubkey + openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com", key, pubkey) doc.content = { self.INCOMING_KEY: True, @@ -198,18 +200,14 @@ class MailReceiver(Service): def _get_owner(self, mail): """ - Given an email, returns the owner (who's delivered to) and the - uuid of the owner. In case of a mailing list mail, the owner - will end up being the mailing list address, but the owner's - uuid will be correct as long as the user exists in the system. + Given an email, returns the uuid of the owner. :param mail: mail to analyze :type mail: email.message.Message - :returns: a tuple of (owner, uuid) - :rtype: tuple(str or None, str or None) + :returns: uuid + :rtype: str or None """ - owner = mail["To"] uuid = None delivereds = mail.get_all("Delivered-To") @@ -220,13 +218,7 @@ class MailReceiver(Service): uuid = parts[0] break - if owner is None: # default to Delivered-To - owner = mail["Delivered-To"] - if owner is None: - log.err("Malformed mail, neither To: nor " - "Delivered-To: field") - - return owner, uuid + return uuid def _process_incoming_email(self, otherself, filepath, mask): """ @@ -247,13 +239,12 @@ class MailReceiver(Service): with filepath.open("r") as f: mail_data = f.read() mail = message_from_string(mail_data) - owner, uuid = self._get_owner(mail) - if owner is None: - # This shouldn't happen, may be a bug in - # postfix? But we don't want to drop mail just - # because of a bug. Skipping this one... + uuid = self._get_owner(mail) + if uuid is None: + log.msg("Don't know how to deliver mail %s, skipping..." % + filepath.path) return - log.msg("Mail owner: %s %s" % (owner, uuid)) + log.msg("Mail owner: %s" % (uuid,)) if uuid is None: log.msg("BUG: There was no uuid!") @@ -261,7 +252,7 @@ class MailReceiver(Service): d = self._users_cdb.getPubKey(uuid) d.addCallbacks(self._encrypt_message, log.err, - (uuid, owner, mail_data)) + (uuid, mail_data)) d.addCallbacks(self._export_message, log.err) d.addCallbacks(self._conditional_remove, log.err, (filepath,)) -- cgit v1.2.3 From db6040781739059d1708fa5471428ed3c0dc2738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Thu, 31 Oct 2013 15:39:45 -0300 Subject: Improve logging to better support unicode Also add some files I forgot to add in a different feature branch. --- changes/VERSION_COMPAT | 1 + changes/bug_improve_logging | 2 ++ changes/bug_look_for_charset_first | 2 ++ src/leap/mx/mail_receiver.py | 18 ++++++++++-------- 4 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 changes/VERSION_COMPAT create mode 100644 changes/bug_improve_logging create mode 100644 changes/bug_look_for_charset_first diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT new file mode 100644 index 0000000..98efc6e --- /dev/null +++ b/changes/VERSION_COMPAT @@ -0,0 +1 @@ +leap.common>=0.3.5 \ No newline at end of file diff --git a/changes/bug_improve_logging b/changes/bug_improve_logging new file mode 100644 index 0000000..920c33b --- /dev/null +++ b/changes/bug_improve_logging @@ -0,0 +1,2 @@ + o Improve logging in general and support possible unicode parameters + without breaking. \ No newline at end of file diff --git a/changes/bug_look_for_charset_first b/changes/bug_look_for_charset_first new file mode 100644 index 0000000..0ab964a --- /dev/null +++ b/changes/bug_look_for_charset_first @@ -0,0 +1,2 @@ + o Try to figure out the encoding of an email first by looking into + its header, if that fails then by using chardet. \ No newline at end of file diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py index 36cfe0b..8fcadce 100644 --- a/src/leap/mx/mail_receiver.py +++ b/src/leap/mx/mail_receiver.py @@ -86,7 +86,7 @@ class MailReceiver(Service): mask = inotify.IN_CREATE for directory, recursive in self._directories: - log.msg("Watching %s --- Recursive: %s" % (directory, recursive)) + log.msg("Watching %r --- Recursive: %r" % (directory, recursive)) self.wm.watch(filepath.FilePath(directory), mask, callbacks=[self._process_incoming_email], recursive=recursive) @@ -109,10 +109,11 @@ class MailReceiver(Service): :rtype: tuple(str, SoledadDocument) """ if uuid is None or pubkey is None or len(pubkey) == 0: + log.msg("_encrypt_message: Something went wrong, here's all " + "I know: %r | %r" % (uuid, pubkey)) return None, None log.msg("Encrypting message to %s's pubkey" % (uuid,)) - log.msg("Pubkey: %s" % (pubkey,)) doc = SoledadDocument(doc_id=str(pyuuid.uuid4())) @@ -165,13 +166,12 @@ class MailReceiver(Service): """ uuid, doc = uuid_doc if uuid is None or doc is None: + log.msg("_export_message: Something went wrong, here's all " + "I know: %r | %r" % (uuid, doc)) return False log.msg("Exporting message for %s" % (uuid,)) - if uuid is None: - uuid = 0 - db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,)) db.put_doc(doc) @@ -192,11 +192,13 @@ class MailReceiver(Service): if do_remove: # remove the original mail try: - log.msg("Removing %s" % (filepath.path,)) + log.msg("Removing %r" % (filepath.path,)) filepath.remove() log.msg("Done removing") except Exception: log.err() + else: + log.msg("Not removing %r" % (filepath.path,)) def _get_owner(self, mail): """ @@ -235,13 +237,13 @@ class MailReceiver(Service): """ try: if os.path.split(filepath.dirname())[-1] == "new": - log.msg("Processing new mail at %s" % (filepath.path,)) + log.msg("Processing new mail at %r" % (filepath.path,)) with filepath.open("r") as f: mail_data = f.read() mail = message_from_string(mail_data) uuid = self._get_owner(mail) if uuid is None: - log.msg("Don't know how to deliver mail %s, skipping..." % + log.msg("Don't know how to deliver mail %r, skipping..." % filepath.path) return log.msg("Mail owner: %s" % (uuid,)) -- cgit v1.2.3 From a3d057708c04d54fe1c3caa8cab2fb55bd592f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Fri, 1 Nov 2013 10:55:01 -0300 Subject: Update requirements --- changes/VERSION_COMPAT | 11 ++++++++++- pkg/requirements.pip | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT index 98efc6e..cc00ecf 100644 --- a/changes/VERSION_COMPAT +++ b/changes/VERSION_COMPAT @@ -1 +1,10 @@ -leap.common>=0.3.5 \ No newline at end of file +################################################# +# This file keeps track of the recent changes +# introduced in internal leap dependencies. +# Add your changes here so we can properly update +# requirements.pip during the release process. +# (leave header when resetting) +################################################# +# +# BEGIN DEPENDENCY LIST ------------------------- +# leap.foo.bar>=x.y.z diff --git a/pkg/requirements.pip b/pkg/requirements.pip index fa5548b..4242ad4 100644 --- a/pkg/requirements.pip +++ b/pkg/requirements.pip @@ -8,6 +8,7 @@ paisley>=0.3.1 # for the time being. couchdb +leap.common>=0.3.5 leap.soledad.common>=0.3.0 leap.keymanager>=0.3.4 -- cgit v1.2.3 From 07e279dbb6540044ae1a657dc9203d6987280469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Fri, 1 Nov 2013 10:56:55 -0300 Subject: Fold in changes --- CHANGELOG | 12 ++++++++++++ changes/bug_fix_ret_codes | 1 - changes/bug_improve_logging | 2 -- changes/bug_look_for_charset_first | 2 -- changes/bug_support_more_than_utf8 | 5 ----- 5 files changed, 12 insertions(+), 10 deletions(-) delete mode 100644 changes/bug_fix_ret_codes delete mode 100644 changes/bug_improve_logging delete mode 100644 changes/bug_look_for_charset_first delete mode 100644 changes/bug_support_more_than_utf8 diff --git a/CHANGELOG b/CHANGELOG index fcc216e..67ff1e5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,15 @@ +0.3.3 Nov 1: + o Fix return codes for check recipient access. Fixes #3356. + o Improve logging in general and support possible unicode parameters + without breaking. + o Try to figure out the encoding of an email first by looking into + its header, if that fails then by using chardet. + o Support more than utf8 encodings for emails. + o Add support for receiving mailing list mails. + o Use the uuid that alias_resolver returned and postfix added to the + mail headers, which improves performance. + o Look for public keys based on uuid instead of mail address. + 0.3.2 Sep 6: o Keep file watcher in memory to prevent losing file events. o Properly save the incoming mail as a doc in couch. diff --git a/changes/bug_fix_ret_codes b/changes/bug_fix_ret_codes deleted file mode 100644 index 54778db..0000000 --- a/changes/bug_fix_ret_codes +++ /dev/null @@ -1 +0,0 @@ - o Fix return codes for check recipient access. Fixes #3356. \ No newline at end of file diff --git a/changes/bug_improve_logging b/changes/bug_improve_logging deleted file mode 100644 index 920c33b..0000000 --- a/changes/bug_improve_logging +++ /dev/null @@ -1,2 +0,0 @@ - o Improve logging in general and support possible unicode parameters - without breaking. \ No newline at end of file diff --git a/changes/bug_look_for_charset_first b/changes/bug_look_for_charset_first deleted file mode 100644 index 0ab964a..0000000 --- a/changes/bug_look_for_charset_first +++ /dev/null @@ -1,2 +0,0 @@ - o Try to figure out the encoding of an email first by looking into - its header, if that fails then by using chardet. \ No newline at end of file diff --git a/changes/bug_support_more_than_utf8 b/changes/bug_support_more_than_utf8 deleted file mode 100644 index 2e467aa..0000000 --- a/changes/bug_support_more_than_utf8 +++ /dev/null @@ -1,5 +0,0 @@ - o Support more than utf8 encodings for emails. - o Add support for receiving mailing list mails. - o Use the uuid that alias_resolver returned and postfix added to the - mail headers, which improves performance. - o Look for public keys based on uuid instead of mail address. \ No newline at end of file -- cgit v1.2.3