diff options
Diffstat (limited to 'src/leap')
| -rw-r--r-- | src/leap/mail/__init__.py | 4 | ||||
| -rw-r--r-- | src/leap/mail/_version.py | 203 | ||||
| -rw-r--r-- | src/leap/mail/imap/fetch.py | 217 | ||||
| -rw-r--r-- | src/leap/mail/imap/server.py | 4 |
4 files changed, 362 insertions, 66 deletions
diff --git a/src/leap/mail/__init__.py b/src/leap/mail/__init__.py index 5f4810c..5b5ba9b 100644 --- a/src/leap/mail/__init__.py +++ b/src/leap/mail/__init__.py @@ -27,3 +27,7 @@ Provide function for loading tests. # def load_tests(): # return unittest.defaultTestLoader.discover('./src/leap/mail') + +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions diff --git a/src/leap/mail/_version.py b/src/leap/mail/_version.py new file mode 100644 index 0000000..8a66c1f --- /dev/null +++ b/src/leap/mail/_version.py @@ -0,0 +1,203 @@ + +IN_LONG_VERSION_PY = True +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (build by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.7+ (https://github.com/warner/python-versioneer) + +# these strings will be replaced by git during git-archive +git_refnames = "$Format:%d$" +git_full = "$Format:%H$" + + +import subprocess +import sys + +def run_command(args, cwd=None, verbose=False): + try: + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) + except EnvironmentError: + e = sys.exc_info()[1] + if verbose: + print("unable to run %s" % args[0]) + print(e) + return None + stdout = p.communicate()[0].strip() + if sys.version >= '3': + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % args[0]) + return None + return stdout + + +import sys +import re +import os.path + +def get_expanded_variables(versionfile_source): + # the code embedded in _version.py can just fetch the value of these + # variables. When used from setup.py, we don't want to import + # _version.py, so we do it with a regexp instead. This function is not + # used from _version.py. + variables = {} + try: + f = open(versionfile_source,"r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + variables["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + variables["full"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return variables + +def versions_from_expanded_variables(variables, tag_prefix, verbose=False): + refnames = variables["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("variables are unexpanded, not using") + return {} # unexpanded, so not in an unpacked git-archive tarball + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs-tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return { "version": r, + "full": variables["full"].strip() } + # no suitable tags, so we use the full revision id + if verbose: + print("no suitable tags, using full revision id") + return { "version": variables["full"].strip(), + "full": variables["full"].strip() } + +def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): + # this runs 'git' from the root of the source tree. That either means + # someone ran a setup.py command (and this code is in versioneer.py, so + # IN_LONG_VERSION_PY=False, thus the containing directory is the root of + # the source tree), or someone ran a project-specific entry point (and + # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the + # containing directory is somewhere deeper in the source tree). This only + # gets called if the git-archive 'subst' variables were *not* expanded, + # and _version.py hasn't already been rewritten with a short version + # string, meaning we're inside a checked out source tree. + + try: + here = os.path.abspath(__file__) + except NameError: + # some py2exe/bbfreeze/non-CPython implementations don't do __file__ + return {} # not always correct + + # versionfile_source is the relative path from the top of the source tree + # (where the .git directory might live) to this file. Invert this to find + # the root from __file__. + root = here + if IN_LONG_VERSION_PY: + for i in range(len(versionfile_source.split("/"))): + root = os.path.dirname(root) + else: + root = os.path.dirname(here) + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %s" % root) + return {} + + GIT = "git" + if sys.platform == "win32": + GIT = "git.cmd" + stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], + cwd=root) + if stdout is None: + return {} + if not stdout.startswith(tag_prefix): + if verbose: + print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) + return {} + tag = stdout[len(tag_prefix):] + stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) + if stdout is None: + return {} + full = stdout.strip() + if tag.endswith("-dirty"): + full += "-dirty" + return {"version": tag, "full": full} + + +def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): + if IN_LONG_VERSION_PY: + # We're running from _version.py. If it's from a source tree + # (execute-in-place), we can work upwards to find the root of the + # tree, and then check the parent directory for a version string. If + # it's in an installed application, there's no hope. + try: + here = os.path.abspath(__file__) + except NameError: + # py2exe/bbfreeze/non-CPython don't have __file__ + return {} # without __file__, we have no hope + # versionfile_source is the relative path from the top of the source + # tree to _version.py. Invert this to find the root from __file__. + root = here + for i in range(len(versionfile_source.split("/"))): + root = os.path.dirname(root) + else: + # we're running from versioneer.py, which means we're running from + # the setup.py in a source tree. sys.argv[0] is setup.py in the root. + here = os.path.abspath(sys.argv[0]) + root = os.path.dirname(here) + + # Source tarballs conventionally unpack into a directory that includes + # both the project name and a version string. + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % + (root, dirname, parentdir_prefix)) + return None + return {"version": dirname[len(parentdir_prefix):], "full": ""} + +tag_prefix = "" +parentdir_prefix = "leap-mail" +versionfile_source = "src/leap/mail/_version.py" + +def get_versions(default={"version": "unknown", "full": ""}, verbose=False): + variables = { "refnames": git_refnames, "full": git_full } + ver = versions_from_expanded_variables(variables, tag_prefix, verbose) + if not ver: + ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) + if not ver: + ver = versions_from_parentdir(parentdir_prefix, versionfile_source, + verbose) + if not ver: + ver = default + return ver + diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py index 2b25d82..8b29c5e 100644 --- a/src/leap/mail/imap/fetch.py +++ b/src/leap/mail/imap/fetch.py @@ -20,10 +20,10 @@ Incoming mail fetcher. import logging import json import ssl +import threading import time from twisted.python import log -from twisted.internet import defer from twisted.internet.task import LoopingCall from twisted.internet.threads import deferToThread @@ -53,6 +53,8 @@ class LeapIncomingMail(object): INCOMING_KEY = "incoming" CONTENT_KEY = "content" + fetching_lock = threading.Lock() + def __init__(self, keymanager, soledad, imap_account, check_period): @@ -95,6 +97,10 @@ class LeapIncomingMail(object): """ self._soledad.create_index("just-mail", "incoming") + # + # Public API: fetch, start_loop, stop. + # + def fetch(self): """ Fetch incoming mail, to be called periodically. @@ -102,9 +108,13 @@ class LeapIncomingMail(object): Calls a deferred that will execute the fetch callback in a separate thread """ - d = deferToThread(self._sync_soledad) - d.addCallbacks(self._process_doclist, self._sync_soledad_err) - return d + if not self.fetching_lock.locked(): + d = deferToThread(self._sync_soledad) + d.addCallbacks(self._signal_fetch_to_ui, self._sync_soledad_error) + d.addCallbacks(self._process_doclist, self._sync_soledad_error) + return d + else: + logger.debug("Already fetching mail.") def start_loop(self): """ @@ -117,52 +127,113 @@ class LeapIncomingMail(object): """ Stops the loop that fetches mail. """ + # XXX should cancel ongoing fetches too. if self._loop and self._loop.running is True: self._loop.stop() + # + # Private methods. + # + + # synchronize incoming mail + def _sync_soledad(self): - log.msg('syncing soledad...') + """ + Synchronizes with remote soledad. - try: + :returns: a list of LeapDocuments, or None. + :rtype: iterable or None + """ + with self.fetching_lock: + log.msg('syncing soledad...') self._soledad.sync() - fetched_ts = time.mktime(time.gmtime()) doclist = self._soledad.get_from_index("just-mail", "*") - num_mails = len(doclist) - log.msg("there are %s mails" % (num_mails,)) - leap_events.signal( - IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) - leap_events.signal( - IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) - return doclist - except ssl.SSLError as exc: - logger.warning('SSL Error while syncing soledad: %r' % (exc,)) - except Exception as exc: - logger.warning('Error while syncing soledad: %r' % (exc,)) + return doclist - def _sync_soledad_err(self, f): - log.err("error syncing soledad: %s" % (f.value,)) - return f + def _signal_fetch_to_ui(self, doclist): + """ + Sends leap events to ui. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: doclist + :rtype: iterable + """ + fetched_ts = time.mktime(time.gmtime()) + num_mails = len(doclist) + log.msg("there are %s mails" % (num_mails,)) + leap_events.signal( + IMAP_FETCHED_INCOMING, str(num_mails), str(fetched_ts)) + leap_events.signal( + IMAP_UNREAD_MAIL, str(self._inbox.getUnseenCount())) + return doclist + + def _sync_soledad_error(self, failure): + """ + Errback for sync errors. + """ + # XXX should signal unrecoverable maybe. + err = failure.value + logger.error("error syncing soledad: %s" % (err,)) + if failure.check(ssl.SSLError): + logger.warning('SSL Error while ' + 'syncing soledad: %r' % (err,)) + elif failure.check(Exception): + logger.warning('Unknown error while ' + 'syncing soledad: %r' % (err,)) def _process_doclist(self, doclist): + """ + Iterates through the doclist, checks if each doc + looks like a message, and yields a deferred that will decrypt and + process the message. + + :param doclist: iterable with msg documents. + :type doclist: iterable. + :returns: a list of deferreds for individual messages. + """ log.msg('processing doclist') if not doclist: logger.debug("no docs found") return num_mails = len(doclist) + + docs_cb = [] for index, doc in enumerate(doclist): logger.debug("processing doc %d of %d: %s" % ( index, num_mails, doc)) leap_events.signal( IMAP_MSG_PROCESSING, str(index), str(num_mails)) keys = doc.content.keys() - if ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys: - - # XXX should check for _enc_scheme == "pubkey" || "none" - # that is what incoming mail uses. + if self._is_msg(keys): + # Ok, this looks like a legit msg. + # Let's process it! encdata = doc.content[ENC_JSON_KEY] - defer.Deferred(self._decrypt_msg(doc, encdata)) + + # Deferred chain for individual messages + d = deferToThread(self._decrypt_msg, doc, encdata) + d.addCallback(self._process_decrypted) + d.addCallback(self._add_message_locally) + docs_cb.append(d) else: + # Ooops, this does not. logger.debug('This does not look like a proper msg.') + return docs_cb + + # + # operations on individual messages + # + + def _is_msg(self, keys): + """ + Checks if the keys of a dictionary match the signature + of the document type we use for messages. + + :param keys: iterable containing the strings to match. + :type keys: iterable of strings. + :rtype: bool + """ + return ENC_SCHEME_KEY in keys and ENC_JSON_KEY in keys def _decrypt_msg(self, doc, encdata): log.msg('decrypting msg') @@ -170,64 +241,80 @@ class LeapIncomingMail(object): try: decrdata = (self._keymanager.decrypt( encdata, key, - # XXX get from public method instead - passphrase=self._soledad._passphrase)) + passphrase=self._soledad.passphrase)) ok = True except Exception as exc: + # XXX move this to errback !!! logger.warning("Error while decrypting msg: %r" % (exc,)) decrdata = "" ok = False leap_events.signal(IMAP_MSG_DECRYPTED, "1" if ok else "0") - # XXX TODO: defer this properly - return self._process_decrypted(doc, decrdata) + return doc, decrdata - def _process_decrypted(self, doc, data): + def _process_decrypted(self, msgtuple): """ Process a successfully decrypted message. - :param doc: a SoledadDocument instance containing the incoming message - :type doc: SoledadDocument - - :param data: the json-encoded, decrypted content of the incoming - message - :type data: str - - :param inbox: a open SoledadMailbox instance where this message is - to be saved - :type inbox: SoledadMailbox + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + :returns: a SoledadDocument and the processed data. + :rtype: (doc, data) """ - log.msg("processing incoming message!") + doc, data = msgtuple msg = json.loads(data) if not isinstance(msg, dict): return False if not msg.get(self.INCOMING_KEY, False): return False + # ok, this is an incoming message rawmsg = msg.get(self.CONTENT_KEY, None) if not rawmsg: return False logger.debug('got incoming message: %s' % (rawmsg,)) + data = self._maybe_decrypt_gpg_msg(rawmsg) + return doc, data - # XXX factor out gpg bits. - try: - pgp_beg = "-----BEGIN PGP MESSAGE-----" - pgp_end = "-----END PGP MESSAGE-----" - if pgp_beg in rawmsg: - first = rawmsg.find(pgp_beg) - last = rawmsg.rfind(pgp_end) - pgp_message = rawmsg[first:first+last] - - decrdata = (self._keymanager.decrypt( - pgp_message, self._pkey, - # XXX get from public method instead - passphrase=self._soledad._passphrase)) - rawmsg = rawmsg.replace(pgp_message, decrdata) - # add to inbox and delete from soledad - self._inbox.addMessage(rawmsg, (self.RECENT_FLAG,)) - leap_events.signal(IMAP_MSG_SAVED_LOCALLY) - doc_id = doc.doc_id - self._soledad.delete_doc(doc) - log.msg("deleted doc %s from incoming" % doc_id) - leap_events.signal(IMAP_MSG_DELETED_INCOMING) - except Exception as e: - logger.error("Problem processing incoming mail: %r" % (e,)) + def _maybe_decrypt_gpg_msg(self, data): + """ + Tries to decrypt a gpg message if data looks like one. + + :param data: the text to be decrypted. + :type data: str + :return: data, possibly descrypted. + :rtype: str + """ + PGP_BEGIN = "-----BEGIN PGP MESSAGE-----" + PGP_END = "-----END PGP MESSAGE-----" + if PGP_BEGIN in data: + begin = data.find(PGP_BEGIN) + end = data.rfind(PGP_END) + pgp_message = data[begin:begin+end] + + decrdata = (self._keymanager.decrypt( + pgp_message, self._pkey, + passphrase=self._soledad.passphrase)) + data = data.replace(pgp_message, decrdata) + return data + + def _add_message_locally(self, msgtuple): + """ + Adds a message to local inbox and delete it from the incoming db + in soledad. + + :param msgtuple: a tuple consisting of a SoledadDocument + instance containing the incoming message + and data, the json-encoded, decrypted content of the + incoming message + :type msgtuple: (SoledadDocument, str) + """ + doc, data = msgtuple + self._inbox.addMessage(data, (self.RECENT_FLAG,)) + leap_events.signal(IMAP_MSG_SAVED_LOCALLY) + doc_id = doc.doc_id + self._soledad.delete_doc(doc) + log.msg("deleted doc %s from incoming" % doc_id) + leap_events.signal(IMAP_MSG_DELETED_INCOMING) diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index a69eb3f..cfcb3d6 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -123,7 +123,9 @@ class IndexedDB(object): if not self._soledad: logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") return - db_indexes = dict(self._soledad.list_indexes()) + db_indexes = dict() + if self._soledad is not None: + db_indexes = dict(self._soledad.list_indexes()) for name, expression in SoledadBackedAccount.INDEXES.items(): if name not in db_indexes: # The index does not yet exist. |