summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--changes/VERSION_COMPAT1
-rw-r--r--changes/bug-4791_url-should-not-end-in-period1
-rw-r--r--changes/bug_4715_fix_message_adding1
-rw-r--r--changes/bug_enqueue-unset-recent2
-rw-r--r--changes/bug_fetch_size4
-rw-r--r--changes/bug_safety-check-for-last-uid1
-rw-r--r--changes/feature_4335_stop-providing-hostname-for-helo1
-rw-r--r--changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted1
-rw-r--r--changes/feaure_4616_fix_mail_indexing1
-rw-r--r--pkg/requirements-testing.pip5
-rw-r--r--src/leap/mail/_version.py215
-rw-r--r--src/leap/mail/imap/fetch.py8
-rw-r--r--src/leap/mail/imap/server.py243
-rwxr-xr-xsrc/leap/mail/imap/tests/imapclient.py7
-rw-r--r--src/leap/mail/imap/tests/test_imap.py152
-rw-r--r--src/leap/mail/messageflow.py25
-rw-r--r--src/leap/mail/smtp/__init__.py6
-rw-r--r--src/leap/mail/smtp/gateway.py5
-rw-r--r--src/leap/mail/smtp/rfc3156.py2
-rw-r--r--src/leap/mail/smtp/tests/test_gateway.py60
-rw-r--r--versioneer.py14
21 files changed, 603 insertions, 152 deletions
diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT
index 032b26a..1d5643f 100644
--- a/changes/VERSION_COMPAT
+++ b/changes/VERSION_COMPAT
@@ -8,4 +8,5 @@
#
# BEGIN DEPENDENCY LIST -------------------------
# leap.foo.bar>=x.y.z
+leap.soledad.client 0.5.0 # get_count_by_index
diff --git a/changes/bug-4791_url-should-not-end-in-period b/changes/bug-4791_url-should-not-end-in-period
new file mode 100644
index 0000000..d4ff29c
--- /dev/null
+++ b/changes/bug-4791_url-should-not-end-in-period
@@ -0,0 +1 @@
+ o Footer url shouldn't end in period. Closes #4791.
diff --git a/changes/bug_4715_fix_message_adding b/changes/bug_4715_fix_message_adding
new file mode 100644
index 0000000..53b875c
--- /dev/null
+++ b/changes/bug_4715_fix_message_adding
@@ -0,0 +1 @@
+ o Soledad writer consumes messages eagerly. Fixes failing tests. Closes: #4715
diff --git a/changes/bug_enqueue-unset-recent b/changes/bug_enqueue-unset-recent
new file mode 100644
index 0000000..8903804
--- /dev/null
+++ b/changes/bug_enqueue-unset-recent
@@ -0,0 +1,2 @@
+ o Enqueue unsetting of recent flag. this was holding the new
+ mails from being displayed soonish.
diff --git a/changes/bug_fetch_size b/changes/bug_fetch_size
new file mode 100644
index 0000000..e9e97b9
--- /dev/null
+++ b/changes/bug_fetch_size
@@ -0,0 +1,4 @@
+ o Limit the size of the messages returned to the IMAP client to 100,
+ since Thunderbird hangs with numbers bigger than those. This is a
+ quick fix until we figure out how does Thunderbird want to receive
+ more than 100 mails at a time. \ No newline at end of file
diff --git a/changes/bug_safety-check-for-last-uid b/changes/bug_safety-check-for-last-uid
new file mode 100644
index 0000000..bb0229f
--- /dev/null
+++ b/changes/bug_safety-check-for-last-uid
@@ -0,0 +1 @@
+ o Sanity check on last_uid setter. Avoids incomplete fetches.
diff --git a/changes/feature_4335_stop-providing-hostname-for-helo b/changes/feature_4335_stop-providing-hostname-for-helo
new file mode 100644
index 0000000..f4b6c29
--- /dev/null
+++ b/changes/feature_4335_stop-providing-hostname-for-helo
@@ -0,0 +1 @@
+ o Stop providing hostname for helo in smtp gateway (#4335).
diff --git a/changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted b/changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted
new file mode 100644
index 0000000..de3bb86
--- /dev/null
+++ b/changes/feature_4671_only-try-to-fetch-keys-for-multipart-signed-or-encrypted
@@ -0,0 +1 @@
+ o Only try to fetch keys for multipart signed or encrypted emails (#4671).
diff --git a/changes/feaure_4616_fix_mail_indexing b/changes/feaure_4616_fix_mail_indexing
new file mode 100644
index 0000000..6e94100
--- /dev/null
+++ b/changes/feaure_4616_fix_mail_indexing
@@ -0,0 +1 @@
+ o Makes efficient use of indexes and count method. Closes: #4616
diff --git a/pkg/requirements-testing.pip b/pkg/requirements-testing.pip
index 7233634..41222f7 100644
--- a/pkg/requirements-testing.pip
+++ b/pkg/requirements-testing.pip
@@ -1,2 +1,7 @@
setuptools-trial
mock
+nose
+rednose
+nose-progressive
+coverage
+pep8>=1.1
diff --git a/src/leap/mail/_version.py b/src/leap/mail/_version.py
index 7267b5b..d80ec47 100644
--- a/src/leap/mail/_version.py
+++ b/src/leap/mail/_version.py
@@ -1,13 +1,210 @@
-# This file was generated by the `freeze_debianver` command in setup.py
-# Using 'versioneer.py' (0.7+) from
-# revision-control system data, or from the parent directory name of an
-# unpacked source archive. Distribution tarballs contain a pre-generated copy
-# of this file.
+IN_LONG_VERSION_PY = True
+# This file helps to compute a version number in source trees obtained from
+# git-archive tarball (such as those provided by githubs download-from-tag
+# feature). Distribution tarballs (build by setup.py sdist) and build
+# directories (produced by setup.py build) will contain a much shorter file
+# that just contains the computed version number.
-version_version = '0.3.8'
-version_full = '790a6f3a5d2edeafcd1de8b4d2d32b861ef24a36'
+# This file is released into the public domain. Generated by
+# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+# these strings will be replaced by git during git-archive
+git_refnames = "$Format:%d$"
+git_full = "$Format:%H$"
-def get_versions(default={}, verbose=False):
- return {'version': version_version, 'full': version_full}
+
+import subprocess
+import sys
+
+
+def run_command(args, cwd=None, verbose=False):
+ try:
+ # remember shell=False, so use git.cmd on windows, not just git
+ p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
+ except EnvironmentError:
+ e = sys.exc_info()[1]
+ if verbose:
+ print("unable to run %s" % args[0])
+ print(e)
+ return None
+ stdout = p.communicate()[0].strip()
+ if sys.version >= '3':
+ stdout = stdout.decode()
+ if p.returncode != 0:
+ if verbose:
+ print("unable to run %s (error)" % args[0])
+ return None
+ return stdout
+
+
+import sys
+import re
+import os.path
+
+
+def get_expanded_variables(versionfile_source):
+ # the code embedded in _version.py can just fetch the value of these
+ # variables. When used from setup.py, we don't want to import
+ # _version.py, so we do it with a regexp instead. This function is not
+ # used from _version.py.
+ variables = {}
+ try:
+ f = open(versionfile_source, "r")
+ for line in f.readlines():
+ if line.strip().startswith("git_refnames ="):
+ mo = re.search(r'=\s*"(.*)"', line)
+ if mo:
+ variables["refnames"] = mo.group(1)
+ if line.strip().startswith("git_full ="):
+ mo = re.search(r'=\s*"(.*)"', line)
+ if mo:
+ variables["full"] = mo.group(1)
+ f.close()
+ except EnvironmentError:
+ pass
+ return variables
+
+
+def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
+ refnames = variables["refnames"].strip()
+ if refnames.startswith("$Format"):
+ if verbose:
+ print("variables are unexpanded, not using")
+ return {} # unexpanded, so not in an unpacked git-archive tarball
+ refs = set([r.strip() for r in refnames.strip("()").split(",")])
+ # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
+ # just "foo-1.0". If we see a "tag: " prefix, prefer those.
+ TAG = "tag: "
+ tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+ if not tags:
+ # Either we're using git < 1.8.3, or there really are no tags. We use
+ # a heuristic: assume all version tags have a digit. The old git %d
+ # expansion behaves like git log --decorate=short and strips out the
+ # refs/heads/ and refs/tags/ prefixes that would let us distinguish
+ # between branches and tags. By ignoring refnames without digits, we
+ # filter out many common branch names like "release" and
+ # "stabilization", as well as "HEAD" and "master".
+ tags = set([r for r in refs if re.search(r'\d', r)])
+ if verbose:
+ print("discarding '%s', no digits" % ",".join(refs - tags))
+ if verbose:
+ print("likely tags: %s" % ",".join(sorted(tags)))
+ for ref in sorted(tags):
+ # sorting will prefer e.g. "2.0" over "2.0rc1"
+ if ref.startswith(tag_prefix):
+ r = ref[len(tag_prefix):]
+ if verbose:
+ print("picking %s" % r)
+ return {"version": r,
+ "full": variables["full"].strip()}
+ # no suitable tags, so we use the full revision id
+ if verbose:
+ print("no suitable tags, using full revision id")
+ return {"version": variables["full"].strip(),
+ "full": variables["full"].strip()}
+
+
+def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
+ # this runs 'git' from the root of the source tree. That either means
+ # someone ran a setup.py command (and this code is in versioneer.py, so
+ # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
+ # the source tree), or someone ran a project-specific entry point (and
+ # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
+ # containing directory is somewhere deeper in the source tree). This only
+ # gets called if the git-archive 'subst' variables were *not* expanded,
+ # and _version.py hasn't already been rewritten with a short version
+ # string, meaning we're inside a checked out source tree.
+
+ try:
+ here = os.path.abspath(__file__)
+ except NameError:
+ # some py2exe/bbfreeze/non-CPython implementations don't do __file__
+ return {} # not always correct
+
+ # versionfile_source is the relative path from the top of the source tree
+ # (where the .git directory might live) to this file. Invert this to find
+ # the root from __file__.
+ root = here
+ if IN_LONG_VERSION_PY:
+ for i in range(len(versionfile_source.split("/"))):
+ root = os.path.dirname(root)
+ else:
+ root = os.path.dirname(here)
+ if not os.path.exists(os.path.join(root, ".git")):
+ if verbose:
+ print("no .git in %s" % root)
+ return {}
+
+ GIT = "git"
+ if sys.platform == "win32":
+ GIT = "git.cmd"
+ stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
+ cwd=root)
+ if stdout is None:
+ return {}
+ if not stdout.startswith(tag_prefix):
+ if verbose:
+ print("tag '%s' doesn't start with prefix '%s'" %
+ (stdout, tag_prefix))
+ return {}
+ tag = stdout[len(tag_prefix):]
+ stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
+ if stdout is None:
+ return {}
+ full = stdout.strip()
+ if tag.endswith("-dirty"):
+ full += "-dirty"
+ return {"version": tag, "full": full}
+
+
+def versions_from_parentdir(parentdir_prefix, versionfile_source,
+ verbose=False):
+ if IN_LONG_VERSION_PY:
+ # We're running from _version.py. If it's from a source tree
+ # (execute-in-place), we can work upwards to find the root of the
+ # tree, and then check the parent directory for a version string. If
+ # it's in an installed application, there's no hope.
+ try:
+ here = os.path.abspath(__file__)
+ except NameError:
+ # py2exe/bbfreeze/non-CPython don't have __file__
+ return {} # without __file__, we have no hope
+ # versionfile_source is the relative path from the top of the source
+ # tree to _version.py. Invert this to find the root from __file__.
+ root = here
+ for i in range(len(versionfile_source.split("/"))):
+ root = os.path.dirname(root)
+ else:
+ # we're running from versioneer.py, which means we're running from
+ # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
+ here = os.path.abspath(sys.argv[0])
+ root = os.path.dirname(here)
+
+ # Source tarballs conventionally unpack into a directory that includes
+ # both the project name and a version string.
+ dirname = os.path.basename(root)
+ if not dirname.startswith(parentdir_prefix):
+ if verbose:
+ print("guessing rootdir is '%s', but '%s' doesn't "
+ "start with prefix '%s'" %
+ (root, dirname, parentdir_prefix))
+ return None
+ return {"version": dirname[len(parentdir_prefix):], "full": ""}
+
+tag_prefix = ""
+parentdir_prefix = "leap-mail"
+versionfile_source = "src/leap/mail/_version.py"
+
+
+def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
+ variables = {"refnames": git_refnames, "full": git_full}
+ ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
+ if not ver:
+ ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
+ if not ver:
+ ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
+ verbose)
+ if not ver:
+ ver = default
+ return ver
diff --git a/src/leap/mail/imap/fetch.py b/src/leap/mail/imap/fetch.py
index 14f7a9b..b1c34ba 100644
--- a/src/leap/mail/imap/fetch.py
+++ b/src/leap/mail/imap/fetch.py
@@ -389,10 +389,12 @@ class LeapIncomingMail(object):
# try to obtain sender public key
senderPubkey = None
fromHeader = msg.get('from', None)
- if fromHeader is not None:
+ if fromHeader is not None \
+ and (msg.get_content_type() == 'multipart/encrypted' \
+ or msg.get_content_type() == 'multipart/signed'):
_, senderAddress = parseaddr(fromHeader)
try:
- senderPubkey = self._keymanager.get_key(
+ senderPubkey = self._keymanager.get_key_from_cache(
senderAddress, OpenPGPKey)
except keymanager_errors.KeyNotFound:
pass
@@ -511,7 +513,7 @@ class LeapIncomingMail(object):
if PGP_BEGIN in data:
begin = data.find(PGP_BEGIN)
end = data.find(PGP_END)
- pgp_message = data[begin:end+len(PGP_END)]
+ pgp_message = data[begin:end + len(PGP_END)]
try:
decrdata, valid_sig = self._decrypt_and_verify_data(
pgp_message, senderPubkey)
diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py
index 6320a51..2739f8c 100644
--- a/src/leap/mail/imap/server.py
+++ b/src/leap/mail/imap/server.py
@@ -31,8 +31,10 @@ from zope.proxy import sameProxiedObjects
from twisted.mail import imap4
from twisted.internet import defer
+from twisted.internet.threads import deferToThread
from twisted.python import log
+
from leap.common import events as leap_events
from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
from leap.common.check import leap_assert, leap_assert_type
@@ -74,6 +76,7 @@ class WithMsgFields(object):
CREATED_KEY = "created"
SUBSCRIBED_KEY = "subscribed"
RW_KEY = "rw"
+ LAST_UID_KEY = "lastuid"
# Document Type, for indexing
TYPE_KEY = "type"
@@ -165,6 +168,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
TYPE_SUBS_IDX = 'by-type-and-subscribed'
TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
+ # Tomas created the `recent and seen index`, but the semantic is not too
+ # correct since the recent flag is volatile.
TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
KTYPE = WithMsgFields.TYPE_KEY
@@ -197,6 +202,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
WithMsgFields.CLOSED_KEY: False,
WithMsgFields.SUBSCRIBED_KEY: False,
WithMsgFields.RW_KEY: 1,
+ WithMsgFields.LAST_UID_KEY: 0
}
def __init__(self, account_name, soledad=None):
@@ -618,14 +624,14 @@ class LeapMessage(WithMsgFields):
Retrieve the flags associated with this message
:return: The flags, represented as strings
- :rtype: iterable
+ :rtype: tuple
"""
if self._doc is None:
return []
flags = self._doc.content.get(self.FLAGS_KEY, None)
if flags:
flags = map(str, flags)
- return flags
+ return tuple(flags)
# setFlags, addFlags, removeFlags are not in the interface spec
# but we use them with store command.
@@ -637,11 +643,12 @@ class LeapMessage(WithMsgFields):
Returns a SoledadDocument that needs to be updated by the caller.
:param flags: the flags to update in the message.
- :type flags: sequence of str
+ :type flags: tuple of str
:return: a SoledadDocument instance
:rtype: SoledadDocument
"""
+ leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
log.msg('setting flags')
doc = self._doc
doc.content[self.FLAGS_KEY] = flags
@@ -656,13 +663,14 @@ class LeapMessage(WithMsgFields):
Returns a SoledadDocument that needs to be updated by the caller.
:param flags: the flags to add to the message.
- :type flags: sequence of str
+ :type flags: tuple of str
:return: a SoledadDocument instance
:rtype: SoledadDocument
"""
+ leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
oldflags = self.getFlags()
- return self.setFlags(list(set(flags + oldflags)))
+ return self.setFlags(tuple(set(flags + oldflags)))
def removeFlags(self, flags):
"""
@@ -671,20 +679,21 @@ class LeapMessage(WithMsgFields):
Returns a SoledadDocument that needs to be updated by the caller.
:param flags: the flags to be removed from the message.
- :type flags: sequence of str
+ :type flags: tuple of str
:return: a SoledadDocument instance
:rtype: SoledadDocument
"""
+ leap_assert(isinstance(flags, tuple), "flags need to be a tuple")
oldflags = self.getFlags()
- return self.setFlags(list(set(oldflags) - set(flags)))
+ return self.setFlags(tuple(set(oldflags) - set(flags)))
def getInternalDate(self):
"""
Retrieve the date internally associated with this message
- @rtype: C{str}
- @retur: An RFC822-formatted date string.
+ :rtype: C{str}
+ :return: An RFC822-formatted date string.
"""
return str(self._doc.content.get(self.DATE_KEY, ''))
@@ -710,8 +719,9 @@ class LeapMessage(WithMsgFields):
:rtype: StringIO
"""
fd = cStringIO.StringIO()
- charset = get_email_charset(self._doc.content.get(self.RAW_KEY, ''))
content = self._doc.content.get(self.RAW_KEY, '')
+ charset = get_email_charset(
+ unicode(self._doc.content.get(self.RAW_KEY, '')))
try:
content = content.encode(charset)
except (UnicodeEncodeError, UnicodeDecodeError) as e:
@@ -736,15 +746,16 @@ class LeapMessage(WithMsgFields):
:rtype: StringIO
"""
fd = StringIO.StringIO()
- charset = get_email_charset(self._doc.content.get(self.RAW_KEY, ''))
content = self._doc.content.get(self.RAW_KEY, '')
+ charset = get_email_charset(
+ unicode(self._doc.content.get(self.RAW_KEY, '')))
try:
content = content.encode(charset)
except (UnicodeEncodeError, UnicodeDecodeError) as e:
logger.error("Unicode error {0}".format(e))
content = content.encode(charset, 'replace')
fd.write(content)
- # SHOULD use a separate BODY FIELD ...
+ # XXX SHOULD use a separate BODY FIELD ...
fd.seek(0)
return fd
@@ -834,14 +845,24 @@ class SoledadDocWriter(object):
"""
self._soledad = soledad
- def consume(self, item):
+ def consume(self, queue):
"""
Creates a new document in soledad db.
- :param item: object to update. content of the document to be inserted.
- :type item: dict
+ :param queue: queue to get item from, with content of the document
+ to be inserted.
+ :type queue: Queue
"""
- self._soledad.create_doc(item)
+ empty = queue.empty()
+ while not empty:
+ item = queue.get()
+ payload = item['payload']
+ mode = item['mode']
+ if mode == "create":
+ self._soledad.create_doc(payload)
+ elif mode == "put":
+ self._soledad.put_doc(payload)
+ empty = queue.empty()
class MessageCollection(WithMsgFields, IndexedDB):
@@ -909,9 +930,9 @@ class MessageCollection(WithMsgFields, IndexedDB):
# to be processed serially by the consumer (the writer). We just
# need to `put` the new material on its plate.
- self._soledad_writer = MessageProducer(
+ self.soledad_writer = MessageProducer(
SoledadDocWriter(soledad),
- period=0.2)
+ period=0.1)
def _get_empty_msg(self):
"""
@@ -941,6 +962,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
:param uid: the message uid for this mailbox
:type uid: int
"""
+ logger.debug('adding message')
if flags is None:
flags = tuple()
leap_assert_type(flags, tuple)
@@ -985,7 +1007,11 @@ class MessageCollection(WithMsgFields, IndexedDB):
# ...should get a sanity check here.
content[self.UID_KEY] = uid
- self._soledad_writer.put(content)
+ logger.debug('enqueuing message for write')
+
+ # XXX create namedtuple
+ self.soledad_writer.put({"mode": "create",
+ "payload": content})
# XXX have to decide what shall we do with errors with this change...
#return self._soledad.create_doc(content)
@@ -1039,6 +1065,8 @@ class MessageCollection(WithMsgFields, IndexedDB):
:param index: the index of the sequence (zero-indexed)
:type index: int
"""
+ # XXX inneficient! ---- we should keep an index document
+ # with uid -- doc_uuid :)
try:
return self.get_all()[index]
except IndexError:
@@ -1064,15 +1092,6 @@ class MessageCollection(WithMsgFields, IndexedDB):
"""
return self.DELETED_FLAG in doc.content[self.FLAGS_KEY]
- def get_last(self):
- """
- Gets the last LeapMessage
- """
- _all = self.get_all()
- if not _all:
- return None
- return LeapMessage(_all[-1])
-
def get_all(self):
"""
Get all message documents for the selected mailbox.
@@ -1089,11 +1108,25 @@ class MessageCollection(WithMsgFields, IndexedDB):
all_docs = [doc for doc in self._soledad.get_from_index(
SoledadBackedAccount.TYPE_MBOX_IDX,
self.TYPE_MESSAGE_VAL, self.mbox)]
- #if not self.is_deleted(doc)]
# highly inneficient, but first let's grok it and then
# let's worry about efficiency.
+
+ # XXX FIXINDEX
return sorted(all_docs, key=lambda item: item.content['uid'])
+ def count(self):
+ """
+ Return the count of messages for this mailbox.
+
+ :rtype: int
+ """
+ count = self._soledad.get_count_from_index(
+ SoledadBackedAccount.TYPE_MBOX_IDX,
+ self.TYPE_MESSAGE_VAL, self.mbox)
+ return count
+
+ # unseen messages
+
def unseen_iter(self):
"""
Get an iterator for the message docs with no `seen` flag
@@ -1103,8 +1136,20 @@ class MessageCollection(WithMsgFields, IndexedDB):
"""
return (doc for doc in
self._soledad.get_from_index(
- SoledadBackedAccount.TYPE_MBOX_RECT_SEEN_IDX,
- self.TYPE_MESSAGE_VAL, self.mbox, '1', '0'))
+ SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
+ self.TYPE_MESSAGE_VAL, self.mbox, '0'))
+
+ def count_unseen(self):
+ """
+ Count all messages with the `Unseen` flag.
+
+ :returns: count
+ :rtype: int
+ """
+ count = self._soledad.get_count_from_index(
+ SoledadBackedAccount.TYPE_MBOX_SEEN_IDX,
+ self.TYPE_MESSAGE_VAL, self.mbox, '0')
+ return count
def get_unseen(self):
"""
@@ -1115,6 +1160,8 @@ class MessageCollection(WithMsgFields, IndexedDB):
"""
return [LeapMessage(doc) for doc in self.unseen_iter()]
+ # recent messages
+
def recent_iter(self):
"""
Get an iterator for the message docs with `recent` flag.
@@ -1136,13 +1183,17 @@ class MessageCollection(WithMsgFields, IndexedDB):
"""
return [LeapMessage(doc) for doc in self.recent_iter()]
- def count(self):
+ def count_recent(self):
"""
- Return the count of messages for this mailbox.
+ Count all messages with the `Recent` flag.
+ :returns: count
:rtype: int
"""
- return len(self.get_all())
+ count = self._soledad.get_count_from_index(
+ SoledadBackedAccount.TYPE_MBOX_RECT_IDX,
+ self.TYPE_MESSAGE_VAL, self.mbox, '1')
+ return count
def __len__(self):
"""
@@ -1172,8 +1223,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
:return: LeapMessage or None if not found.
:rtype: LeapMessage
"""
- #try:
- #return self.get_msg_by_uid(uid)
+ # XXX FIXME inneficcient, we are evaulating.
try:
return [doc
for doc in self.get_all()][uid - 1]
@@ -1245,7 +1295,7 @@ class SoledadMailbox(WithMsgFields):
self._soledad = soledad
self.messages = MessageCollection(
- mbox=mbox, soledad=soledad)
+ mbox=mbox, soledad=self._soledad)
if not self.getFlags():
self.setFlags(self.INIT_FLAGS)
@@ -1360,6 +1410,47 @@ class SoledadMailbox(WithMsgFields):
closed = property(
_get_closed, _set_closed, doc="Closed attribute.")
+ def _get_last_uid(self):
+ """
+ Return the last uid for this mailbox.
+
+ :return: the last uid for messages in this mailbox
+ :rtype: bool
+ """
+ mbox = self._get_mbox()
+ return mbox.content.get(self.LAST_UID_KEY, 1)
+
+ def _set_last_uid(self, uid):
+ """
+ Sets the last uid for this mailbox.
+
+ :param uid: the uid to be set
+ :type uid: int
+ """
+ leap_assert(isinstance(uid, int), "uid has to be int")
+ mbox = self._get_mbox()
+ key = self.LAST_UID_KEY
+
+ count = self.getMessageCount()
+
+ # XXX safety-catch. If we do get duplicates,
+ # we want to avoid further duplication.
+
+ if uid >= count:
+ value = uid
+ else:
+ # something is wrong,
+ # just set the last uid
+ # beyond the max msg count.
+ logger.debug("WRONG uid < count. Setting last uid to ", count)
+ value = count
+
+ mbox.content[key] = value
+ self._soledad.put_doc(mbox)
+
+ last_uid = property(
+ _get_last_uid, _set_last_uid, doc="Last_UID attribute.")
+
def getUIDValidity(self):
"""
Return the unique validity identifier for this mailbox.
@@ -1389,17 +1480,18 @@ class SoledadMailbox(WithMsgFields):
def getUIDNext(self):
"""
Return the likely UID for the next message added to this
- mailbox. Currently it returns the current length incremented
- by one.
+ mailbox. Currently it returns the higher UID incremented by
+ one.
+
+ We increment the next uid *each* time this function gets called.
+ In this way, there will be gaps if the message with the allocated
+ uid cannot be saved. But that is preferable to having race conditions
+ if we get to parallel message adding.
:rtype: int
"""
- last = self.messages.get_last()
- if last:
- nextuid = last.getUID() + 1
- else:
- nextuid = 1
- return nextuid
+ self.last_uid += 1
+ return self.last_uid
def getMessageCount(self):
"""
@@ -1416,7 +1508,7 @@ class SoledadMailbox(WithMsgFields):
:return: count of messages flagged `unseen`
:rtype: int
"""
- return len(self.messages.get_unseen())
+ return self.messages.count_unseen()
def getRecentCount(self):
"""
@@ -1425,7 +1517,7 @@ class SoledadMailbox(WithMsgFields):
:return: count of messages flagged `recent`
:rtype: int
"""
- return len(self.messages.get_recent())
+ return self.messages.count_recent()
def isWriteable(self):
"""
@@ -1482,6 +1574,7 @@ class SoledadMailbox(WithMsgFields):
"""
# XXX we should treat the message as an IMessage from here
uid_next = self.getUIDNext()
+ logger.debug('Adding msg with UID :%s' % uid_next)
if flags is None:
flags = tuple()
else:
@@ -1490,8 +1583,11 @@ class SoledadMailbox(WithMsgFields):
self.messages.add_msg(message, flags=flags, date=date,
uid=uid_next)
- exists = len(self.messages)
- recent = len(self.messages.get_recent())
+ exists = self.getMessageCount()
+ recent = self.getRecentCount()
+ logger.debug("there are %s messages, %s recent" % (
+ exists,
+ recent))
for listener in self.listeners:
listener.newMessages(exists, recent)
return defer.succeed(None)
@@ -1518,9 +1614,9 @@ class SoledadMailbox(WithMsgFields):
"""
if not self.isWriteable():
raise imap4.ReadOnlyMailbox
-
delete = []
deleted = []
+
for m in self.messages.get_all():
if self.DELETED_FLAG in m.content[self.FLAGS_KEY]:
delete.append(m)
@@ -1557,12 +1653,7 @@ class SoledadMailbox(WithMsgFields):
iter(messages)
except TypeError:
# looks like we cannot iterate
- last = self.messages.get_last()
- if last is None:
- uid_last = 1
- else:
- uid_last = last.getUID()
- messages.last = uid_last
+ messages.last = self.last_uid
# for sequence numbers (uid = 0)
if sequence:
@@ -1581,14 +1672,39 @@ class SoledadMailbox(WithMsgFields):
else:
print "fetch %s, no msg found!!!" % msg_id
- return tuple(result)
+ if self.isWriteable():
+ self._unset_recent_flag()
+
+ return tuple(result[:100])
+
+ def _unset_recent_flag(self):
+ """
+ Unsets `Recent` flag from a tuple of messages.
+ Called from fetch.
+
+ From RFC, about `Recent`:
+
+ Message is "recently" arrived in this mailbox. This session
+ is the first session to have been notified about this
+ message; if the session is read-write, subsequent sessions
+ will not see \Recent set for this message. This flag can not
+ be altered by the client.
+
+ If it is not possible to determine whether or not this
+ session is the first session to be notified about a message,
+ then that message SHOULD be considered recent.
+ """
+ log.msg('unsetting recent flags...')
+ for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()):
+ newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,))
+ self._update(newflags)
def _signal_unread_to_ui(self):
"""
Sends unread event to ui.
"""
- leap_events.signal(
- IMAP_UNREAD_MAIL, str(self.getUnseenCount()))
+ unseen = self.getUnseenCount()
+ leap_events.signal(IMAP_UNREAD_MAIL, str(unseen))
def store(self, messages, flags, mode, uid):
"""
@@ -1620,6 +1736,10 @@ class SoledadMailbox(WithMsgFields):
read-write.
"""
# XXX implement also sequence (uid = 0)
+ # XXX we should prevent cclient from setting Recent flag.
+ leap_assert(not isinstance(flags, basestring),
+ "flags cannot be a string")
+ flags = tuple(flags)
if not self.isWriteable():
log.msg('read only mailbox!')
@@ -1664,8 +1784,9 @@ class SoledadMailbox(WithMsgFields):
"""
Updates document in u1db database
"""
- #log.msg('updating doc... %s ' % doc)
- self._soledad.put_doc(doc)
+ # XXX create namedtuple
+ self.messages.soledad_writer.put({"mode": "put",
+ "payload": doc})
def __repr__(self):
"""
diff --git a/src/leap/mail/imap/tests/imapclient.py b/src/leap/mail/imap/tests/imapclient.py
index 027396c..c353cee 100755
--- a/src/leap/mail/imap/tests/imapclient.py
+++ b/src/leap/mail/imap/tests/imapclient.py
@@ -21,7 +21,7 @@ from twisted.python import log
class TrivialPrompter(basic.LineReceiver):
- #from os import linesep as delimiter
+ # from os import linesep as delimiter
promptDeferred = None
@@ -42,6 +42,7 @@ class TrivialPrompter(basic.LineReceiver):
class SimpleIMAP4Client(imap4.IMAP4Client):
+
"""
Add callbacks when the client receives greeting messages from
an IMAP server.
@@ -98,8 +99,8 @@ def cbServerGreeting(proto, username, password):
# Try to authenticate securely
return proto.authenticate(
password).addCallback(
- cbAuthentication, proto).addErrback(
- ebAuthentication, proto, username, password)
+ cbAuthentication, proto).addErrback(
+ ebAuthentication, proto, username, password)
def ebConnection(reason):
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index ca73a11..f87b534 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -25,6 +25,7 @@ XXX add authors from the original twisted tests.
@license: GPLv3, see included LICENSE file
"""
# XXX review license of the original tests!!!
+from nose.twistedtools import deferred
try:
from cStringIO import StringIO
@@ -54,7 +55,7 @@ import twisted.cred.credentials
import twisted.cred.portal
-#import u1db
+# import u1db
from leap.common.testing.basetest import BaseLeapTest
from leap.mail.imap.server import SoledadMailbox
@@ -120,17 +121,19 @@ def initialize_soledad(email, gnupg_home, tempdir):
return _soledad
-##########################################
+#
# Simple LEAP IMAP4 Server for testing
-##########################################
+#
class SimpleLEAPServer(imap4.IMAP4Server):
+
"""
A Simple IMAP4 Server with mailboxes backed by Soledad.
This should be pretty close to the real LeapIMAP4Server that we
will be instantiating as a service, minus the authentication bits.
"""
+
def __init__(self, *args, **kw):
soledad = kw.pop('soledad', None)
@@ -153,7 +156,7 @@ class SimpleLEAPServer(imap4.IMAP4Server):
def lineReceived(self, line):
if self.timeoutTest:
- #Do not send a respones
+ # Do not send a respones
return
imap4.IMAP4Server.lineReceived(self, line)
@@ -168,6 +171,7 @@ class SimpleLEAPServer(imap4.IMAP4Server):
class TestRealm:
+
"""
A minimal auth realm for testing purposes only
"""
@@ -177,12 +181,13 @@ class TestRealm:
return imap4.IAccount, self.theAccount, lambda: None
-######################################
+#
# Simple IMAP4 Client for testing
-######################################
+#
class SimpleClient(imap4.IMAP4Client):
+
"""
A Simple IMAP4 Client to test our
Soledad-LEAPServer
@@ -210,6 +215,7 @@ class SimpleClient(imap4.IMAP4Client):
class IMAP4HelperMixin(BaseLeapTest):
+
"""
MixIn containing several utilities to be shared across
different TestCases
@@ -245,13 +251,13 @@ class IMAP4HelperMixin(BaseLeapTest):
# Soledad: config info
cls.gnupg_home = "%s/gnupg" % cls.tempdir
cls.email = 'leap@leap.se'
- #cls.db1_file = "%s/db1.u1db" % cls.tempdir
- #cls.db2_file = "%s/db2.u1db" % cls.tempdir
+ # cls.db1_file = "%s/db1.u1db" % cls.tempdir
+ # cls.db2_file = "%s/db2.u1db" % cls.tempdir
# open test dbs
- #cls._db1 = u1db.open(cls.db1_file, create=True,
- #document_factory=SoledadDocument)
- #cls._db2 = u1db.open(cls.db2_file, create=True,
- #document_factory=SoledadDocument)
+ # cls._db1 = u1db.open(cls.db1_file, create=True,
+ # document_factory=SoledadDocument)
+ # cls._db2 = u1db.open(cls.db2_file, create=True,
+ # document_factory=SoledadDocument)
# initialize soledad by hand so we can control keys
cls._soledad = initialize_soledad(
@@ -261,7 +267,7 @@ class IMAP4HelperMixin(BaseLeapTest):
# now we're passing the mailbox name, so we
# should get this into a partial or something.
- #cls.sm = SoledadMailbox("mailbox", soledad=cls._soledad)
+ # cls.sm = SoledadMailbox("mailbox", soledad=cls._soledad)
# XXX REFACTOR --- self.server (in setUp) is initializing
# a SoledadBackedAccount
@@ -273,8 +279,8 @@ class IMAP4HelperMixin(BaseLeapTest):
Restores the old path and home environment variables.
Removes the temporal dir created for tests.
"""
- #cls._db1.close()
- #cls._db2.close()
+ # cls._db1.close()
+ # cls._db2.close()
cls._soledad.close()
os.environ["PATH"] = cls.old_path
@@ -328,8 +334,8 @@ class IMAP4HelperMixin(BaseLeapTest):
acct.delete(mb)
# FIXME add again
- #for subs in acct.subscriptions:
- #acct.unsubscribe(subs)
+ # for subs in acct.subscriptions:
+ # acct.unsubscribe(subs)
del self.server
del self.client
@@ -364,7 +370,11 @@ class IMAP4HelperMixin(BaseLeapTest):
def _ebGeneral(self, failure):
self.client.transport.loseConnection()
self.server.transport.loseConnection()
- log.err(failure, "Problem with %r" % (self.function,))
+ # can we do something similar?
+ # I guess this was ok with trial, but not in noseland...
+ #log.err(failure, "Problem with %r" % (self.function,))
+ raise failure.value
+ #failure.trap(Exception)
def loopback(self):
return loopback.loopbackAsync(self.server, self.client)
@@ -375,16 +385,20 @@ class IMAP4HelperMixin(BaseLeapTest):
#
class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
+
"""
Tests for the MessageCollection class
"""
+
def setUp(self):
"""
setUp method for each test
We override mixin method since we are only testing
MessageCollection interface in this particular TestCase
"""
- self.messages = MessageCollection("testmbox", self._soledad._db)
+ self.messages = MessageCollection("testmbox", self._soledad)
+ for m in self.messages.get_all():
+ self.messages.remove(m)
def tearDown(self):
"""
@@ -414,13 +428,63 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
})
self.assertEqual(self.messages.count(), 0)
+ def testMultipleAdd(self):
+ """
+ Add multiple messages
+ """
+ mc = self.messages
+ self.assertEqual(self.messages.count(), 0)
+ mc.add_msg('Stuff', subject="test1")
+ self.assertEqual(self.messages.count(), 1)
+ mc.add_msg('Stuff', subject="test2")
+ self.assertEqual(self.messages.count(), 2)
+ mc.add_msg('Stuff', subject="test3")
+ self.assertEqual(self.messages.count(), 3)
+ mc.add_msg('Stuff', subject="test4")
+ self.assertEqual(self.messages.count(), 4)
+ mc.add_msg('Stuff', subject="test5")
+ mc.add_msg('Stuff', subject="test6")
+ mc.add_msg('Stuff', subject="test7")
+ mc.add_msg('Stuff', subject="test8")
+ mc.add_msg('Stuff', subject="test9")
+ mc.add_msg('Stuff', subject="test10")
+ self.assertEqual(self.messages.count(), 10)
+
+ def testRecentCount(self):
+ """
+ Test the recent count
+ """
+ mc = self.messages
+ self.assertEqual(self.messages.count_recent(), 0)
+ mc.add_msg('Stuff', subject="test1", uid=1)
+ # For the semantics defined in the RFC, we auto-add the
+ # recent flag by default.
+ self.assertEqual(self.messages.count_recent(), 1)
+ mc.add_msg('Stuff', subject="test2", uid=2, flags=('\\Deleted',))
+ self.assertEqual(self.messages.count_recent(), 2)
+ mc.add_msg('Stuff', subject="test3", uid=3, flags=('\\Recent',))
+ self.assertEqual(self.messages.count_recent(), 3)
+ mc.add_msg('Stuff', subject="test4", uid=4,
+ flags=('\\Deleted', '\\Recent'))
+ self.assertEqual(self.messages.count_recent(), 4)
+
+ for m in mc:
+ msg = self.messages.get_msg_by_uid(m.get('uid'))
+ msg_newflags = msg.removeFlags(('\\Recent',))
+ self._soledad.put_doc(msg_newflags)
+
+ self.assertEqual(mc.count_recent(), 0)
+
def testFilterByMailbox(self):
"""
Test that queries filter by selected mailbox
"""
mc = self.messages
+ self.assertEqual(self.messages.count(), 0)
mc.add_msg('', subject="test1")
+ self.assertEqual(self.messages.count(), 1)
mc.add_msg('', subject="test2")
+ self.assertEqual(self.messages.count(), 2)
mc.add_msg('', subject="test3")
self.assertEqual(self.messages.count(), 3)
@@ -434,6 +498,7 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
+
"""
Tests for the generic behavior of the LeapIMAP4Server
which, right now, it's just implemented in this test file as
@@ -451,6 +516,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
# mailboxes operations
#
+ @deferred(timeout=None)
def testCreate(self):
"""
Test whether we can create mailboxes
@@ -489,6 +555,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
answers.sort()
self.assertEqual(mbox, [a.upper() for a in answers])
+ @deferred(timeout=None)
def testDelete(self):
"""
Test whether we can delete mailboxes
@@ -537,6 +604,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
failure.Failure)))
return d
+ @deferred(timeout=None)
def testNonExistentDelete(self):
"""
Test what happens if we try to delete a non-existent mailbox.
@@ -562,6 +630,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
'No such mailbox'))
return d
+ @deferred(timeout=None)
def testIllegalDelete(self):
"""
Try deleting a mailbox with sub-folders, and \NoSelect flag set.
@@ -596,6 +665,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.assertEqual(str(self.failure.value), expected))
return d
+ @deferred(timeout=None)
def testRename(self):
"""
Test whether we can rename a mailbox
@@ -619,6 +689,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
['NEWNAME']))
return d
+ @deferred(timeout=None)
def testIllegalInboxRename(self):
"""
Try to rename inbox. We expect it to fail. Then it would be not
@@ -646,6 +717,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.stashed, failure.Failure)))
return d
+ @deferred(timeout=None)
def testHierarchicalRename(self):
"""
Try to rename hierarchical mailboxes
@@ -672,6 +744,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
mboxes.sort()
self.assertEqual(mboxes, [s.upper() for s in expected])
+ @deferred(timeout=None)
def testSubscribe(self):
"""
Test whether we can mark a mailbox as subscribed to
@@ -693,6 +766,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
['THIS/MBOX']))
return d
+ @deferred(timeout=None)
def testUnsubscribe(self):
"""
Test whether we can unsubscribe from a set of mailboxes
@@ -717,6 +791,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
['THAT/MBOX']))
return d
+ @deferred(timeout=None)
def testSelect(self):
"""
Try to select a mailbox
@@ -756,6 +831,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
# capabilities
#
+ @deferred(timeout=None)
def testCapability(self):
caps = {}
@@ -771,6 +847,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return d.addCallback(lambda _: self.assertEqual(expected, caps))
+ @deferred(timeout=None)
def testCapabilityWithAuth(self):
caps = {}
self.server.challengers[
@@ -795,6 +872,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
# authentication
#
+ @deferred(timeout=None)
def testLogout(self):
"""
Test log out
@@ -809,6 +887,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
d = self.loopback()
return d.addCallback(lambda _: self.assertEqual(self.loggedOut, 1))
+ @deferred(timeout=None)
def testNoop(self):
"""
Test noop command
@@ -824,6 +903,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
d = self.loopback()
return d.addCallback(lambda _: self.assertEqual(self.responses, []))
+ @deferred(timeout=None)
def testLogin(self):
"""
Test login
@@ -840,6 +920,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.assertEqual(self.server.account, SimpleLEAPServer.theAccount)
self.assertEqual(self.server.state, 'auth')
+ @deferred(timeout=None)
def testFailedLogin(self):
"""
Test bad login
@@ -858,6 +939,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.assertEqual(self.server.account, None)
self.assertEqual(self.server.state, 'unauth')
+ @deferred(timeout=None)
def testLoginRequiringQuoting(self):
"""
Test login requiring quoting
@@ -882,6 +964,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
# Inspection
#
+ @deferred(timeout=None)
def testNamespace(self):
"""
Test retrieving namespace
@@ -906,6 +989,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
[[['', '/']], [], []]))
return d
+ @deferred(timeout=None)
def testExamine(self):
"""
L{IMAP4Client.examine} issues an I{EXAMINE} command to the server and
@@ -975,6 +1059,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
d2 = self.loopback()
return defer.gatherResults([d1, d2]).addCallback(lambda _: self.listed)
+ @deferred(timeout=None)
def testList(self):
"""
Test List command
@@ -991,22 +1076,21 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
))
return d
- # XXX implement subscriptions
- '''
+ @deferred(timeout=None)
def testLSub(self):
"""
Test LSub command
"""
- SimpleLEAPServer.theAccount.subscribe('ROOT/SUBTHINGL')
+ SimpleLEAPServer.theAccount.subscribe('ROOT/SUBTHINGL2')
def lsub():
return self.client.lsub('root', '%')
d = self._listSetup(lsub)
d.addCallback(self.assertEqual,
- [(SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL")])
+ [(SoledadMailbox.INIT_FLAGS, "/", "ROOT/SUBTHINGL2")])
return d
- '''
+ @deferred(timeout=None)
def testStatus(self):
"""
Test Status command
@@ -1038,6 +1122,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
))
return d
+ @deferred(timeout=None)
def testFailedStatus(self):
"""
Test failed status command with a non-existent mailbox
@@ -1077,6 +1162,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
# messages
#
+ @deferred(timeout=None)
def testFullAppend(self):
"""
Test appending a full message to the mailbox
@@ -1117,6 +1203,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
+ @deferred(timeout=None)
def testPartialAppend(self):
"""
Test partially appending a message to the mailbox
@@ -1143,19 +1230,22 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
d1.addCallbacks(self._cbStopClient, self._ebGeneral)
d2 = self.loopback()
d = defer.gatherResults([d1, d2])
- return d.addCallback(self._cbTestPartialAppend, infile)
+ return d.addCallback(
+ self._cbTestPartialAppend, infile)
def _cbTestPartialAppend(self, ignored, infile):
mb = SimpleLEAPServer.theAccount.getMailbox('PARTIAL/SUBTHING')
+
self.assertEqual(1, len(mb.messages))
self.assertEqual(
- ['\\SEEN',],
+ ['\\SEEN', ],
mb.messages[1].content['flags']
)
self.assertEqual(
'Right now', mb.messages[1].content['date'])
self.assertEqual(open(infile).read(), mb.messages[1].content['raw'])
+ @deferred(timeout=None)
def testCheck(self):
"""
Test check command
@@ -1179,6 +1269,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
# Okay, that was fun
+ @deferred(timeout=None)
def testClose(self):
"""
Test closing the mailbox. We expect to get deleted all messages flagged
@@ -1214,9 +1305,9 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
self.assertEqual(
m.messages[1].content['subject'],
'Message 2')
-
self.failUnless(m.closed)
+ @deferred(timeout=None)
def testExpunge(self):
"""
Test expunge command
@@ -1226,8 +1317,11 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
m = SimpleLEAPServer.theAccount.getMailbox(name)
m.messages.add_msg('', subject="Message 1",
flags=('\\Deleted', 'AnotherFlag'))
+ self.failUnless(m.messages.count() == 1)
m.messages.add_msg('', subject="Message 2", flags=('AnotherFlag',))
+ self.failUnless(m.messages.count() == 2)
m.messages.add_msg('', subject="Message 3", flags=('\\Deleted',))
+ self.failUnless(m.messages.count() == 3)
def login():
return self.client.login('testuser', 'password-test')
@@ -1253,7 +1347,8 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
return d.addCallback(self._cbTestExpunge, m)
def _cbTestExpunge(self, ignored, m):
- self.assertEqual(len(m.messages), 1)
+ # we only left 1 mssage with no deleted flag
+ self.assertEqual(m.messages.count(), 1)
self.assertEqual(
m.messages[1].content['subject'],
'Message 2')
@@ -1262,6 +1357,7 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
class IMAP4ServerSearchTestCase(IMAP4HelperMixin, unittest.TestCase):
+
"""
Tests for the behavior of the search_* functions in L{imap4.IMAP4Server}.
"""
diff --git a/src/leap/mail/messageflow.py b/src/leap/mail/messageflow.py
index 21f6d62..a0a571d 100644
--- a/src/leap/mail/messageflow.py
+++ b/src/leap/mail/messageflow.py
@@ -26,11 +26,11 @@ from zope.interface import Interface, implements
class IMessageConsumer(Interface):
- def consume(self, item):
+ def consume(self, queue):
"""
Consumes the passed item.
- :param item: an object to be consumed.
+ :param item: q queue where we put the object to be consumed.
:type item: object
"""
# TODO we could add an optional type to be passed
@@ -44,11 +44,12 @@ class DummyMsgConsumer(object):
implements(IMessageConsumer)
- def consume(self, item):
+ def consume(self, queue):
"""
Just prints the passed item.
"""
- print "got item %s" % item
+ if not queue.empty():
+ print "got item %s" % queue.get()
class MessageProducer(object):
@@ -97,14 +98,9 @@ class MessageProducer(object):
If the queue is found empty, the loop is stopped. It will be started
again after the addition of new items.
"""
- # XXX right now I'm assuming that the period is good enough to allow
- # a right pace of processing. but we could also pass the queue object
- # to the consumer and let it choose whether process a new item or not.
-
+ self._consumer.consume(self._queue)
if self._queue.empty():
self.stop()
- else:
- self._consumer.consume(self._queue.get())
# public methods
@@ -114,20 +110,19 @@ class MessageProducer(object):
If the queue was empty, we will start the loop again.
"""
- was_empty = self._queue.empty()
-
# XXX this might raise if the queue does not accept any new
# items. what to do then?
self._queue.put(item)
- if was_empty:
- self.start()
+ self.start()
def start(self):
"""
Starts polling for new items.
"""
if not self._loop.running:
- self._loop.start(self._period)
+ self._loop.start(self._period, now=True)
+ else:
+ print "was running..., not starting"
def stop(self):
"""
diff --git a/src/leap/mail/smtp/__init__.py b/src/leap/mail/smtp/__init__.py
index d3eb9e8..bbd4064 100644
--- a/src/leap/mail/smtp/__init__.py
+++ b/src/leap/mail/smtp/__init__.py
@@ -30,7 +30,7 @@ from leap.mail.smtp.gateway import SMTPFactory
def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
- smtp_cert, smtp_key, encrypted_only):
+ smtp_cert, smtp_key, encrypted_only):
"""
Setup SMTP gateway to run with Twisted.
@@ -52,8 +52,8 @@ def setup_smtp_gateway(port, userid, keymanager, smtp_host, smtp_port,
:type smtp_cert: str
:param smtp_key: The client key for authentication.
:type smtp_key: str
- :param encrypted_only: Whether the SMTP gateway should send unencrypted mail
- or not.
+ :param encrypted_only: Whether the SMTP gateway should send unencrypted
+ mail or not.
:type encrypted_only: bool
:returns: tuple of SMTPFactory, twisted.internet.tcp.Port
diff --git a/src/leap/mail/smtp/gateway.py b/src/leap/mail/smtp/gateway.py
index a78bd55..bef5c6d 100644
--- a/src/leap/mail/smtp/gateway.py
+++ b/src/leap/mail/smtp/gateway.py
@@ -52,6 +52,7 @@ from leap.common.events import proto, signal
from leap.keymanager import KeyManager
from leap.keymanager.openpgp import OpenPGPKey
from leap.keymanager.errors import KeyNotFound
+from leap.mail import __version__
from leap.mail.smtp.rfc3156 import (
MultipartSigned,
MultipartEncrypted,
@@ -492,7 +493,7 @@ class EncryptedMessage(object):
heloFallback=True,
requireAuthentication=False,
requireTransportSecurity=True)
- factory.domain = LOCAL_FQDN
+ factory.domain = __version__
signal(proto.SMTP_SEND_MESSAGE_START, self._user.dest.addrstr)
reactor.connectSSL(
self._host, self._port, factory,
@@ -603,7 +604,7 @@ class EncryptedMessage(object):
from_address = validate_address(self._fromAddress.addrstr)
username, domain = from_address.split('@')
self.lines.append('--')
- self.lines.append('%s - https://%s/key/%s.' %
+ self.lines.append('%s - https://%s/key/%s' %
(self.FOOTER_STRING, domain, username))
self.lines.append('')
self._origmsg = self.parseMessage()
diff --git a/src/leap/mail/smtp/rfc3156.py b/src/leap/mail/smtp/rfc3156.py
index dd48475..9739531 100644
--- a/src/leap/mail/smtp/rfc3156.py
+++ b/src/leap/mail/smtp/rfc3156.py
@@ -361,7 +361,7 @@ class PGPSignature(MIMEApplication):
"""
def __init__(self, _data, name='signature.asc'):
MIMEApplication.__init__(self, _data, 'pgp-signature',
- _encoder=lambda x: x, name=name)
+ _encoder=lambda x: x, name=name)
self.add_header('Content-Description', 'OpenPGP Digital Signature')
diff --git a/src/leap/mail/smtp/tests/test_gateway.py b/src/leap/mail/smtp/tests/test_gateway.py
index 4c2f04f..88ee5f7 100644
--- a/src/leap/mail/smtp/tests/test_gateway.py
+++ b/src/leap/mail/smtp/tests/test_gateway.py
@@ -101,10 +101,16 @@ class TestSmtpGateway(TestCaseWithKeyManager):
'250 Sender address accepted',
'250 Recipient address accepted',
'354 Continue']
- proto = SMTPFactory(u'anotheruser@leap.se',
- self._km, self._config['host'], self._config['port'],
+
+ # XXX this bit can be refactored away in a helper
+ # method...
+ proto = SMTPFactory(
+ u'anotheruser@leap.se',
+ self._km, self._config['host'],
+ self._config['port'],
self._config['cert'], self._config['key'],
self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
+ # snip...
transport = proto_helpers.StringTransport()
proto.makeConnection(transport)
for i, line in enumerate(self.EMAIL_DATA):
@@ -118,8 +124,10 @@ class TestSmtpGateway(TestCaseWithKeyManager):
"""
Test if message gets encrypted to destination email.
"""
- proto = SMTPFactory(u'anotheruser@leap.se',
- self._km, self._config['host'], self._config['port'],
+ proto = SMTPFactory(
+ u'anotheruser@leap.se',
+ self._km, self._config['host'],
+ self._config['port'],
self._config['cert'], self._config['key'],
self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
fromAddr = Address(ADDRESS_2)
@@ -129,7 +137,8 @@ class TestSmtpGateway(TestCaseWithKeyManager):
self._config['port'], self._config['cert'], self._config['key'])
for line in self.EMAIL_DATA[4:12]:
m.lineReceived(line)
- m.eomReceived()
+ #m.eomReceived() # this includes a defer, so we avoid calling it here
+ m.lines.append('') # add a trailing newline
# we need to call the following explicitelly because it was deferred
# inside the previous method
m._maybe_encrypt_and_sign()
@@ -149,7 +158,7 @@ class TestSmtpGateway(TestCaseWithKeyManager):
m._msg.get_payload(1).get_payload(), privkey)
self.assertEqual(
'\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' +
- 'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n',
+ 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n',
decrypted,
'Decrypted text differs from plaintext.')
@@ -158,8 +167,10 @@ class TestSmtpGateway(TestCaseWithKeyManager):
Test if message gets encrypted to destination email and signed with
sender key.
"""
- proto = SMTPFactory(u'anotheruser@leap.se',
- self._km, self._config['host'], self._config['port'],
+ proto = SMTPFactory(
+ u'anotheruser@leap.se',
+ self._km, self._config['host'],
+ self._config['port'],
self._config['cert'], self._config['key'],
self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
user = User(ADDRESS, 'gateway.leap.se', proto, ADDRESS)
@@ -170,7 +181,8 @@ class TestSmtpGateway(TestCaseWithKeyManager):
for line in self.EMAIL_DATA[4:12]:
m.lineReceived(line)
# trigger encryption and signing
- m.eomReceived()
+ #m.eomReceived() # this includes a defer, so we avoid calling it here
+ m.lines.append('') # add a trailing newline
# we need to call the following explicitelly because it was deferred
# inside the previous method
m._maybe_encrypt_and_sign()
@@ -192,7 +204,7 @@ class TestSmtpGateway(TestCaseWithKeyManager):
m._msg.get_payload(1).get_payload(), privkey, verify=pubkey)
self.assertEqual(
'\n' + '\r\n'.join(self.EMAIL_DATA[9:12]) + '\r\n\r\n--\r\n' +
- 'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n',
+ 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n',
decrypted,
'Decrypted text differs from plaintext.')
@@ -202,11 +214,14 @@ class TestSmtpGateway(TestCaseWithKeyManager):
"""
# mock the key fetching
self._km.fetch_keys_from_server = Mock(return_value=[])
- proto = SMTPFactory(u'anotheruser@leap.se',
- self._km, self._config['host'], self._config['port'],
+ proto = SMTPFactory(
+ u'anotheruser@leap.se',
+ self._km, self._config['host'],
+ self._config['port'],
self._config['cert'], self._config['key'],
self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
- user = User('ihavenopubkey@nonleap.se', 'gateway.leap.se', proto, ADDRESS)
+ user = User('ihavenopubkey@nonleap.se',
+ 'gateway.leap.se', proto, ADDRESS)
fromAddr = Address(ADDRESS_2)
m = EncryptedMessage(
fromAddr, user, self._km, self._config['host'],
@@ -214,7 +229,8 @@ class TestSmtpGateway(TestCaseWithKeyManager):
for line in self.EMAIL_DATA[4:12]:
m.lineReceived(line)
# trigger signing
- m.eomReceived()
+ #m.eomReceived() # this includes a defer, so we avoid calling it here
+ m.lines.append('') # add a trailing newline
# we need to call the following explicitelly because it was deferred
# inside the previous method
m._maybe_encrypt_and_sign()
@@ -226,8 +242,8 @@ class TestSmtpGateway(TestCaseWithKeyManager):
self.assertEqual('pgp-sha512', m._msg.get_param('micalg'))
# assert content of message
self.assertEqual(
- '\r\n'.join(self.EMAIL_DATA[9:13])+'\r\n--\r\n' +
- 'I prefer encrypted email - https://leap.se/key/anotheruser.\r\n',
+ '\r\n'.join(self.EMAIL_DATA[9:13]) + '\r\n--\r\n' +
+ 'I prefer encrypted email - https://leap.se/key/anotheruser\r\n',
m._msg.get_payload(0).get_payload(decode=True))
# assert content of signature
self.assertTrue(
@@ -262,8 +278,10 @@ class TestSmtpGateway(TestCaseWithKeyManager):
# mock the key fetching
self._km.fetch_keys_from_server = Mock(return_value=[])
# prepare the SMTP factory
- proto = SMTPFactory(u'anotheruser@leap.se',
- self._km, self._config['host'], self._config['port'],
+ proto = SMTPFactory(
+ u'anotheruser@leap.se',
+ self._km, self._config['host'],
+ self._config['port'],
self._config['cert'], self._config['key'],
self._config['encrypted_only']).buildProtocol(('127.0.0.1', 0))
transport = proto_helpers.StringTransport()
@@ -291,8 +309,10 @@ class TestSmtpGateway(TestCaseWithKeyManager):
# mock the key fetching
self._km.fetch_keys_from_server = Mock(return_value=[])
# prepare the SMTP factory with encrypted only equal to false
- proto = SMTPFactory(u'anotheruser@leap.se',
- self._km, self._config['host'], self._config['port'],
+ proto = SMTPFactory(
+ u'anotheruser@leap.se',
+ self._km, self._config['host'],
+ self._config['port'],
self._config['cert'], self._config['key'],
False).buildProtocol(('127.0.0.1', 0))
transport = proto_helpers.StringTransport()
diff --git a/versioneer.py b/versioneer.py
index 34e4807..4e2c0a5 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -115,7 +115,7 @@ import sys
def run_command(args, cwd=None, verbose=False):
try:
- # remember shell=False, so use git.cmd on windows, not just git
+ # remember shell=False, so use git.exe on windows, not just git
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
except EnvironmentError:
e = sys.exc_info()[1]
@@ -230,7 +230,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
GIT = "git"
if sys.platform == "win32":
- GIT = "git.cmd"
+ GIT = "git.exe"
stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
cwd=root)
if stdout is None:
@@ -305,7 +305,7 @@ import sys
def run_command(args, cwd=None, verbose=False):
try:
- # remember shell=False, so use git.cmd on windows, not just git
+ # remember shell=False, so use git.exe on windows, not just git
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
except EnvironmentError:
e = sys.exc_info()[1]
@@ -420,7 +420,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
GIT = "git"
if sys.platform == "win32":
- GIT = "git.cmd"
+ GIT = "git.exe"
stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
cwd=root)
if stdout is None:
@@ -476,7 +476,7 @@ import sys
def do_vcs_install(versionfile_source, ipy):
GIT = "git"
if sys.platform == "win32":
- GIT = "git.cmd"
+ GIT = "git.exe"
run_command([GIT, "add", "versioneer.py"])
run_command([GIT, "add", versionfile_source])
run_command([GIT, "add", ipy])
@@ -489,13 +489,13 @@ def do_vcs_install(versionfile_source, ipy):
present = True
f.close()
except EnvironmentError:
- pass
+ pass
if not present:
f = open(".gitattributes", "a+")
f.write("%s export-subst\n" % versionfile_source)
f.close()
run_command([GIT, "add", ".gitattributes"])
-
+
SHORT_VERSION_PY = """
# This file was generated by 'versioneer.py' (0.7+) from
generated by cgit v1.2.3 (git 2.25.1) at 2024-09-22 07:18:23 +0000