summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--CHANGELOG12
-rw-r--r--changes/VERSION_COMPAT10
-rw-r--r--pkg/requirements.pip7
-rw-r--r--setup.py70
-rw-r--r--src/leap/mx/check_recipient_access.py8
-rw-r--r--src/leap/mx/couchdbhelper.py74
-rw-r--r--src/leap/mx/mail_receiver.py213
7 files changed, 266 insertions, 128 deletions
diff --git a/CHANGELOG b/CHANGELOG
index fcc216e..67ff1e5 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,15 @@
+0.3.3 Nov 1:
+ o Fix return codes for check recipient access. Fixes #3356.
+ o Improve logging in general and support possible unicode parameters
+ without breaking.
+ o Try to figure out the encoding of an email first by looking into
+ its header, if that fails then by using chardet.
+ o Support more than utf8 encodings for emails.
+ o Add support for receiving mailing list mails.
+ o Use the uuid that alias_resolver returned and postfix added to the
+ mail headers, which improves performance.
+ o Look for public keys based on uuid instead of mail address.
+
0.3.2 Sep 6:
o Keep file watcher in memory to prevent losing file events.
o Properly save the incoming mail as a doc in couch.
diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT
new file mode 100644
index 0000000..cc00ecf
--- /dev/null
+++ b/changes/VERSION_COMPAT
@@ -0,0 +1,10 @@
+#################################################
+# This file keeps track of the recent changes
+# introduced in internal leap dependencies.
+# Add your changes here so we can properly update
+# requirements.pip during the release process.
+# (leave header when resetting)
+#################################################
+#
+# BEGIN DEPENDENCY LIST -------------------------
+# leap.foo.bar>=x.y.z
diff --git a/pkg/requirements.pip b/pkg/requirements.pip
index d5db275..4242ad4 100644
--- a/pkg/requirements.pip
+++ b/pkg/requirements.pip
@@ -8,7 +8,8 @@ paisley>=0.3.1
# for the time being.
couchdb
-## XXX change me to whatever you name the package in pypi
-python-gnupg>=0.3.0
+leap.common>=0.3.5
leap.soledad.common>=0.3.0
-leap.keymanager>=0.2.0
+leap.keymanager>=0.3.4
+
+cchardet # we fallback to chardet if this is not available, but it's preferred
diff --git a/setup.py b/setup.py
index 3de66cf..6fec416 100644
--- a/setup.py
+++ b/setup.py
@@ -18,6 +18,7 @@
setup file for leap.mx
"""
import os
+import re
from setuptools import setup, find_packages
import versioneer
@@ -43,6 +44,68 @@ trove_classifiers = [
'Topic :: Security :: Cryptography',
]
+DOWNLOAD_BASE = ('https://github.com/leapcode/leap_mx/'
+ 'archive/%s.tar.gz')
+_versions = versioneer.get_versions()
+VERSION = _versions['version']
+VERSION_FULL = _versions['full']
+DOWNLOAD_URL = ""
+
+# get the short version for the download url
+_version_short = re.findall('\d+\.\d+\.\d+', VERSION)
+if len(_version_short) > 0:
+ VERSION_SHORT = _version_short[0]
+ DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT
+
+cmdclass = versioneer.get_cmdclass()
+
+
+from setuptools import Command
+
+
+class freeze_debianver(Command):
+ """
+ Freezes the version in a debian branch.
+ To be used after merging the development branch onto the debian one.
+ """
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ proceed = str(raw_input(
+ "This will overwrite the file _version.py. Continue? [y/N] "))
+ if proceed != "y":
+ print("He. You scared. Aborting.")
+ return
+ template = r"""
+# This file was generated by the `freeze_debianver` command in setup.py
+# Using 'versioneer.py' (0.7+) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+version_version = '{version}'
+version_full = '{version_full}'
+"""
+ templatefun = r"""
+
+def get_versions(default={}, verbose=False):
+ return {'version': version_version, 'full': version_full}
+"""
+ subst_template = template.format(
+ version=VERSION_SHORT,
+ version_full=VERSION_FULL) + templatefun
+ with open(versioneer.versionfile_source, 'w') as f:
+ f.write(subst_template)
+
+
+cmdclass["freeze_debianver"] = freeze_debianver
+
if os.environ.get("VIRTUAL_ENV", None):
data_files = None
else:
@@ -54,12 +117,15 @@ else:
("/etc/init.d/", ["pkg/leap_mx"])]
setup(
name='leap.mx',
- version=versioneer.get_version(),
- cmdclass=versioneer.get_cmdclass(),
+ version=VERSION,
+ cmdclass=cmdclass,
url="http://github.com/leapcode/leap_mx",
+ download_url=DOWNLOAD_URL,
license='AGPLv3+',
author='The LEAP Encryption Access Project',
author_email='info@leap.se',
+ maintainer='Kali Kaneko',
+ maintainer_email='kali@leap.se',
description=("An asynchronous, transparently-encrypting remailer "
"for the LEAP platform"),
long_description=(
diff --git a/src/leap/mx/check_recipient_access.py b/src/leap/mx/check_recipient_access.py
index 0520c7c..b80ccfd 100644
--- a/src/leap/mx/check_recipient_access.py
+++ b/src/leap/mx/check_recipient_access.py
@@ -29,11 +29,13 @@ from leap.mx.alias_resolver import AliasResolverFactory
class LEAPPostFixTCPMapserverAccess(postfix.PostfixTCPMapServer):
def _cbGot(self, value):
+ # For more info, see:
+ # http://www.postfix.org/tcp_table.5.html
+ # http://www.postfix.org/access.5.html
if value is None:
- self.sendCode(500, postfix.quote("NOT FOUND SORRY"))
+ self.sendCode(500, postfix.quote("REJECT"))
else:
- # We do not send the value in this case
- self.sendCode(200)
+ self.sendCode(200, postfix.quote("OK"))
class CheckRecipientAccessFactory(AliasResolverFactory):
diff --git a/src/leap/mx/couchdbhelper.py b/src/leap/mx/couchdbhelper.py
index 147e6f9..41604ba 100644
--- a/src/leap/mx/couchdbhelper.py
+++ b/src/leap/mx/couchdbhelper.py
@@ -29,7 +29,6 @@ except ImportError:
print "for instructions on getting required dependencies."
try:
- from twisted.internet import defer
from twisted.python import log
except ImportError:
print "This software requires Twisted. Please see the README file"
@@ -49,16 +48,16 @@ class ConnectedCouchDB(client.CouchDB):
"""
Connect to a CouchDB instance.
- @param host: A hostname string for the CouchDB server.
- @type host: str
- @param port: The port of the CouchDB server.
- @type port: int
- @param dbName: (optional) The default database to bind queries to.
- @type dbName: str
- @param username: (optional) The username for authorization.
- @type username: str
- @param str password: (optional) The password for authorization.
- @type password: str
+ :param host: A hostname string for the CouchDB server.
+ :type host: str
+ :param port: The port of the CouchDB server.
+ :type port: int
+ :param dbName: (optional) The default database to bind queries to.
+ :type dbName: str
+ :param username: (optional) The username for authorization.
+ :type username: str
+ :param str password: (optional) The password for authorization.
+ :type password: str
"""
client.CouchDB.__init__(self,
host,
@@ -78,8 +77,8 @@ class ConnectedCouchDB(client.CouchDB):
"""
Callback for listDB that prints the available databases
- @param data: response from the listDB command
- @type data: array
+ :param data: response from the listDB command
+ :type data: array
"""
log.msg("Available databases:")
for database in data:
@@ -101,10 +100,10 @@ class ConnectedCouchDB(client.CouchDB):
"""
Check to see if a particular email or alias exists.
- @param alias: A string representing the email or alias to check.
- @type alias: str
- @return: a deferred for this query
- @rtype twisted.defer.Deferred
+ :param alias: A string representing the email or alias to check.
+ :type alias: str
+ :return: a deferred for this query
+ :rtype twisted.defer.Deferred
"""
assert isinstance(address, (str, unicode)), "Email or alias queries must be string"
@@ -124,12 +123,12 @@ class ConnectedCouchDB(client.CouchDB):
"""
Parses the result of the by_address query and gets the uuid
- @param address: alias looked up
- @type address: string
- @param result: result dictionary
- @type result: dict
- @return: The uuid for alias if available
- @rtype: str
+ :param address: alias looked up
+ :type address: string
+ :param result: result dictionary
+ :type result: dict
+ :return: The uuid for alias if available
+ :rtype: str
"""
for row in result["rows"]:
if row["key"] == address:
@@ -140,20 +139,39 @@ class ConnectedCouchDB(client.CouchDB):
return uuid
return None
- def getPubKey(self, address):
+ def getPubKey(self, uuid):
+ """
+ Returns a deferred that will return the pubkey for the uuid provided
+
+ :param uuid: uuid for the user to query
+ :type uuid: str
+
+ :rtype: Deferred
+ """
d = self.openView(docId="Identity",
viewId="pgp_key_by_email/",
- key=address,
+ user_id=uuid,
reduce=False,
include_docs=True)
- d.addCallbacks(partial(self._get_pgp_key, address), log.err)
+ d.addCallbacks(partial(self._get_pgp_key, uuid), log.err)
return d
- def _get_pgp_key(self, address, result):
+ def _get_pgp_key(self, uuid, result):
+ """
+ Callback used to filter the correct pubkey from the result of
+ the query to the couchdb
+
+ :param uuid: uuid for the user that was queried
+ :type uuid: str
+ :param result: result dictionary for the db query
+ :type result: dict
+
+ :rtype: str or None
+ """
for row in result["rows"]:
- if row["key"] == address:
+ if row["doc"]["user_id"] == uuid:
return row["value"]
return None
diff --git a/src/leap/mx/mail_receiver.py b/src/leap/mx/mail_receiver.py
index 5875034..8fcadce 100644
--- a/src/leap/mx/mail_receiver.py
+++ b/src/leap/mx/mail_receiver.py
@@ -25,14 +25,20 @@ import uuid as pyuuid
import json
import email.utils
+import socket
+
+try:
+ import cchardet as chardet
+except ImportError:
+ import chardet
from email import message_from_string
from twisted.application.service import Service
from twisted.internet import inotify
-from twisted.internet.defer import DeferredList
from twisted.python import filepath, log
+from leap.common.mail import get_email_charset
from leap.soledad.common.document import SoledadDocument
from leap.soledad.common.crypto import (
EncryptionSchemes,
@@ -54,19 +60,20 @@ class MailReceiver(Service):
"""
Constructor
- @param mail_couch_url: URL prefix for the couchdb where mail
+ :param mail_couch_url: URL prefix for the couchdb where mail
should be stored
- @type mail_couch_url: str
- @param users_cdb: CouchDB instance from where to get the uuid
+ :type mail_couch_url: str
+ :param users_cdb: CouchDB instance from where to get the uuid
and pubkey for a user
- @type users_cdb: ConnectedCouchDB
- @param directories: list of directories to monitor
- @type directories: list of tuples (path: str, recursive: bool)
+ :type users_cdb: ConnectedCouchDB
+ :param directories: list of directories to monitor
+ :type directories: list of tuples (path: str, recursive: bool)
"""
# Service doesn't define an __init__
self._mail_couch_url = mail_couch_url
self._users_cdb = users_cdb
self._directories = directories
+ self._domain = socket.gethostbyaddr(socket.gethostname())[0]
def startService(self):
"""
@@ -79,56 +86,49 @@ class MailReceiver(Service):
mask = inotify.IN_CREATE
for directory, recursive in self._directories:
- log.msg("Watching %s --- Recursive: %s" % (directory, recursive))
+ log.msg("Watching %r --- Recursive: %r" % (directory, recursive))
self.wm.watch(filepath.FilePath(directory), mask,
callbacks=[self._process_incoming_email],
recursive=recursive)
- def _gather_uuid_pubkey(self, results):
- if len(results) < 2:
- return None, None
-
- # DeferredList results are structured like this:
- # [ (succeeded, pubkey), (succeeded, uuid) ]
- # succeeded is a bool value that specifies if the
- # corresponding callback succeeded
- pubkey_res, uuid_res = results
-
- pubkey = pubkey_res[1] if pubkey_res[0] else None
- uuid = uuid_res[1] if uuid_res[0] else None
-
- return uuid, pubkey
-
- def _encrypt_message(self, uuid_pubkey, address, message):
+ def _encrypt_message(self, pubkey, uuid, message):
"""
- Given a UUID, a public key, address and a message, it encrypts
- the message to that public key.
+ Given a UUID, a public key and a message, it encrypts the
+ message to that public key.
The address is needed in order to build the OpenPGPKey object.
- @param uuid_pubkey: tuple that holds the uuid and the public
- key as it is returned by the previous call in the chain
- @type uuid_pubkey: tuple (str, str)
- @param address: mail address for this message
- @type address: str
- @param message: message contents
- @type message: str
+ :param uuid_pubkey: tuple that holds the uuid and the public
+ key as it is returned by the previous call in the
+ chain
+ :type uuid_pubkey: tuple (str, str)
+ :param message: message contents
+ :type message: str
- @return: uuid, doc to sync with Soledad
- @rtype: tuple(str, SoledadDocument)
+ :return: uuid, doc to sync with Soledad or None, None if
+ something went wrong.
+ :rtype: tuple(str, SoledadDocument)
"""
- uuid, pubkey = uuid_pubkey
+ if uuid is None or pubkey is None or len(pubkey) == 0:
+ log.msg("_encrypt_message: Something went wrong, here's all "
+ "I know: %r | %r" % (uuid, pubkey))
+ return None, None
+
log.msg("Encrypting message to %s's pubkey" % (uuid,))
- log.msg("Pubkey: %s" % (pubkey,))
doc = SoledadDocument(doc_id=str(pyuuid.uuid4()))
+ encoding = get_email_charset(message, default=None)
+ if encoding is None:
+ result = chardet.detect(message)
+ encoding = result["encoding"]
+
data = {'incoming': True, 'content': message}
if pubkey is None or len(pubkey) == 0:
doc.content = {
self.INCOMING_KEY: True,
ENC_SCHEME_KEY: EncryptionSchemes.NONE,
- ENC_JSON_KEY: json.dumps(data)
+ ENC_JSON_KEY: json.dumps(data, encoding=encoding)
}
return uuid, doc
@@ -136,13 +136,15 @@ class MailReceiver(Service):
with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg:
gpg.import_keys(pubkey)
key = gpg.list_keys().pop()
- openpgp_key = openpgp._build_key_from_gpg(address, key, pubkey)
+ # We don't care about the actual address, so we use a
+ # dummy one, we just care about the import of the pubkey
+ openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com", key, pubkey)
doc.content = {
self.INCOMING_KEY: True,
ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
ENC_JSON_KEY: str(gpg.encrypt(
- json.dumps(data),
+ json.dumps(data, encoding=encoding),
openpgp_key.fingerprint,
symmetric=False))
}
@@ -153,20 +155,22 @@ class MailReceiver(Service):
"""
Given a UUID and a SoledadDocument, it saves it directly in the
couchdb that serves as a backend for Soledad, in a db
- accessible to the recipient of the mail
+ accessible to the recipient of the mail.
- @param uuid_doc: tuple that holds the UUID and SoledadDocument
- @type uuid_doc: tuple(str, SoledadDocument)
+ :param uuid_doc: tuple that holds the UUID and SoledadDocument
+ :type uuid_doc: tuple(str, SoledadDocument)
- @return: True if it's ok to remove the message, False
- otherwise
- @rtype: bool
+ :return: True if it's ok to remove the message, False
+ otherwise
+ :rtype: bool
"""
uuid, doc = uuid_doc
- log.msg("Exporting message for %s" % (uuid,))
+ if uuid is None or doc is None:
+ log.msg("_export_message: Something went wrong, here's all "
+ "I know: %r | %r" % (uuid, doc))
+ return False
- if uuid is None:
- uuid = 0
+ log.msg("Exporting message for %s" % (uuid,))
db = CouchDatabase(self._mail_couch_url, "user-%s" % (uuid,))
db.put_doc(doc)
@@ -177,58 +181,83 @@ class MailReceiver(Service):
def _conditional_remove(self, do_remove, filepath):
"""
- Removes the message if do_remove is True
+ Removes the message if do_remove is True.
- @param do_remove: True if the message should be removed, False
- otherwise
- @type do_remove: bool
- @param filepath: path to the mail
- @type filepath: twisted.python.filepath.FilePath
+ :param do_remove: True if the message should be removed, False
+ otherwise
+ :type do_remove: bool
+ :param filepath: path to the mail
+ :type filepath: twisted.python.filepath.FilePath
"""
if do_remove:
# remove the original mail
try:
- log.msg("Removing %s" % (filepath.path,))
+ log.msg("Removing %r" % (filepath.path,))
filepath.remove()
log.msg("Done removing")
- except:
+ except Exception:
log.err()
+ else:
+ log.msg("Not removing %r" % (filepath.path,))
+
+ def _get_owner(self, mail):
+ """
+ Given an email, returns the uuid of the owner.
+
+ :param mail: mail to analyze
+ :type mail: email.message.Message
+
+ :returns: uuid
+ :rtype: str or None
+ """
+ uuid = None
+
+ delivereds = mail.get_all("Delivered-To")
+ for to in delivereds:
+ name, addr = email.utils.parseaddr(to)
+ parts = addr.split("@")
+ if len(parts) > 1 and parts[1] == self._domain:
+ uuid = parts[0]
+ break
+
+ return uuid
def _process_incoming_email(self, otherself, filepath, mask):
"""
- Callback that processes incoming email
-
- @param otherself: Watch object for the current callback from
- inotify
- @type otherself: twisted.internet.inotify._Watch
- @param filepath: Path of the file that changed
- @type filepath: twisted.python.filepath.FilePath
- @param mask: identifier for the type of change that triggered
- this callback
- @type mask: int
+ Callback that processes incoming email.
+
+ :param otherself: Watch object for the current callback from
+ inotify.
+ :type otherself: twisted.internet.inotify._Watch
+ :param filepath: Path of the file that changed
+ :type filepath: twisted.python.filepath.FilePath
+ :param mask: identifier for the type of change that triggered
+ this callback
+ :type mask: int
"""
- if os.path.split(filepath.dirname())[-1] == "new":
- log.msg("Processing new mail at %s" % (filepath.path,))
- with filepath.open("r") as f:
- mail_data = f.read()
- mail = message_from_string(mail_data)
- owner = mail["To"]
- if owner is None: # default to Delivered-To
- owner = mail["Delivered-To"]
- if owner is None:
- log.err("Malformed mail, neither To: nor "
- "Delivered-To: field")
- log.msg("Mail owner: %s" % (owner,))
-
- owner = email.utils.parseaddr(owner)[1]
- log.msg("%s received a new mail" % (owner,))
- dpubk = self._users_cdb.getPubKey(owner)
- duuid = self._users_cdb.queryByAddress(owner)
- d = DeferredList([dpubk, duuid])
- d.addCallbacks(self._gather_uuid_pubkey, log.err)
- d.addCallbacks(self._encrypt_message, log.err,
- (owner, mail_data))
- d.addCallbacks(self._export_message, log.err)
- d.addCallbacks(self._conditional_remove, log.err,
- (filepath,))
- d.addErrback(log.err)
+ try:
+ if os.path.split(filepath.dirname())[-1] == "new":
+ log.msg("Processing new mail at %r" % (filepath.path,))
+ with filepath.open("r") as f:
+ mail_data = f.read()
+ mail = message_from_string(mail_data)
+ uuid = self._get_owner(mail)
+ if uuid is None:
+ log.msg("Don't know how to deliver mail %r, skipping..." %
+ filepath.path)
+ return
+ log.msg("Mail owner: %s" % (uuid,))
+
+ if uuid is None:
+ log.msg("BUG: There was no uuid!")
+ return
+
+ d = self._users_cdb.getPubKey(uuid)
+ d.addCallbacks(self._encrypt_message, log.err,
+ (uuid, mail_data))
+ d.addCallbacks(self._export_message, log.err)
+ d.addCallbacks(self._conditional_remove, log.err,
+ (filepath,))
+ d.addErrback(log.err)
+ except Exception:
+ log.err()