From ddad3391ba8ad611a9bdaaf689b408d44eec9cc6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 11 Dec 2013 12:11:21 -0400 Subject: consume messages eagerly --- src/leap/mail/imap/server.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 6320a51..73ec223 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -834,14 +834,19 @@ class SoledadDocWriter(object): """ self._soledad = soledad - def consume(self, item): + def consume(self, queue): """ Creates a new document in soledad db. - :param item: object to update. content of the document to be inserted. - :type item: dict + :param queue: queue to get item from, with content of the document + to be inserted. + :type queue: Queue """ - self._soledad.create_doc(item) + empty = queue.empty() + while not empty: + item = queue.get() + self._soledad.create_doc(item) + empty = queue.empty() class MessageCollection(WithMsgFields, IndexedDB): @@ -911,7 +916,7 @@ class MessageCollection(WithMsgFields, IndexedDB): self._soledad_writer = MessageProducer( SoledadDocWriter(soledad), - period=0.2) + period=0.1) def _get_empty_msg(self): """ @@ -941,6 +946,7 @@ class MessageCollection(WithMsgFields, IndexedDB): :param uid: the message uid for this mailbox :type uid: int """ + logger.debug('adding message') if flags is None: flags = tuple() leap_assert_type(flags, tuple) @@ -985,6 +991,7 @@ class MessageCollection(WithMsgFields, IndexedDB): # ...should get a sanity check here. content[self.UID_KEY] = uid + logger.debug('enqueuing message for write') self._soledad_writer.put(content) # XXX have to decide what shall we do with errors with this change... #return self._soledad.create_doc(content) @@ -1518,9 +1525,9 @@ class SoledadMailbox(WithMsgFields): """ if not self.isWriteable(): raise imap4.ReadOnlyMailbox - delete = [] deleted = [] + for m in self.messages.get_all(): if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: delete.append(m) -- cgit v1.2.3 From d1719ca40f6c7838fb41915706960c822f081237 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 5 Dec 2013 11:24:23 -0400 Subject: count_foo uses expanded u1db count method. Other fixes in the commit: * Correct the semantic for the recent flag (reset) * Minor unicode fixes. * Use a field for tracking the last_uid In general, this tries to squash all the quick and naive methods that were relying on evaluating all the message objects before returning a result. Further work is still needed, planned also for 0.5 release. get_by_index needs to be indexed too. --- src/leap/mail/imap/server.py | 184 +++++++++++++++++++++++++++++++------------ 1 file changed, 135 insertions(+), 49 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 73ec223..b79e691 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -74,6 +74,7 @@ class WithMsgFields(object): CREATED_KEY = "created" SUBSCRIBED_KEY = "subscribed" RW_KEY = "rw" + LAST_UID_KEY = "lastuid" # Document Type, for indexing TYPE_KEY = "type" @@ -165,6 +166,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): TYPE_SUBS_IDX = 'by-type-and-subscribed' TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' + # Tomas created the `recent and seen index`, but the semantic is not too + # correct since the recent flag is volatile. TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' KTYPE = WithMsgFields.TYPE_KEY @@ -197,6 +200,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): WithMsgFields.CLOSED_KEY: False, WithMsgFields.SUBSCRIBED_KEY: False, WithMsgFields.RW_KEY: 1, + WithMsgFields.LAST_UID_KEY: 0 } def __init__(self, account_name, soledad=None): @@ -618,14 +622,14 @@ class LeapMessage(WithMsgFields): Retrieve the flags associated with this message :return: The flags, represented as strings - :rtype: iterable + :rtype: tuple """ if self._doc is None: return [] flags = self._doc.content.get(self.FLAGS_KEY, None) if flags: flags = map(str, flags) - return flags + return tuple(flags) # setFlags, addFlags, removeFlags are not in the interface spec # but we use them with store command. @@ -637,11 +641,12 @@ class LeapMessage(WithMsgFields): Returns a SoledadDocument that needs to be updated by the caller. :param flags: the flags to update in the message. - :type flags: sequence of str + :type flags: tuple of str :return: a SoledadDocument instance :rtype: SoledadDocument """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") log.msg('setting flags') doc = self._doc doc.content[self.FLAGS_KEY] = flags @@ -656,13 +661,14 @@ class LeapMessage(WithMsgFields): Returns a SoledadDocument that needs to be updated by the caller. :param flags: the flags to add to the message. - :type flags: sequence of str + :type flags: tuple of str :return: a SoledadDocument instance :rtype: SoledadDocument """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") oldflags = self.getFlags() - return self.setFlags(list(set(flags + oldflags))) + return self.setFlags(tuple(set(flags + oldflags))) def removeFlags(self, flags): """ @@ -671,20 +677,21 @@ class LeapMessage(WithMsgFields): Returns a SoledadDocument that needs to be updated by the caller. :param flags: the flags to be removed from the message. - :type flags: sequence of str + :type flags: tuple of str :return: a SoledadDocument instance :rtype: SoledadDocument """ + leap_assert(isinstance(flags, tuple), "flags need to be a tuple") oldflags = self.getFlags() - return self.setFlags(list(set(oldflags) - set(flags))) + return self.setFlags(tuple(set(oldflags) - set(flags))) def getInternalDate(self): """ Retrieve the date internally associated with this message - @rtype: C{str} - @retur: An RFC822-formatted date string. + :rtype: C{str} + :return: An RFC822-formatted date string. """ return str(self._doc.content.get(self.DATE_KEY, '')) @@ -710,8 +717,9 @@ class LeapMessage(WithMsgFields): :rtype: StringIO """ fd = cStringIO.StringIO() - charset = get_email_charset(self._doc.content.get(self.RAW_KEY, '')) content = self._doc.content.get(self.RAW_KEY, '') + charset = get_email_charset( + unicode(self._doc.content.get(self.RAW_KEY, ''))) try: content = content.encode(charset) except (UnicodeEncodeError, UnicodeDecodeError) as e: @@ -736,8 +744,9 @@ class LeapMessage(WithMsgFields): :rtype: StringIO """ fd = StringIO.StringIO() - charset = get_email_charset(self._doc.content.get(self.RAW_KEY, '')) content = self._doc.content.get(self.RAW_KEY, '') + charset = get_email_charset( + unicode(self._doc.content.get(self.RAW_KEY, ''))) try: content = content.encode(charset) except (UnicodeEncodeError, UnicodeDecodeError) as e: @@ -1046,6 +1055,8 @@ class MessageCollection(WithMsgFields, IndexedDB): :param index: the index of the sequence (zero-indexed) :type index: int """ + # XXX inneficient! ---- we should keep an index document + # with uid -- doc_uuid :) try: return self.get_all()[index] except IndexError: @@ -1071,15 +1082,6 @@ class MessageCollection(WithMsgFields, IndexedDB): """ return self.DELETED_FLAG in doc.content[self.FLAGS_KEY] - def get_last(self): - """ - Gets the last LeapMessage - """ - _all = self.get_all() - if not _all: - return None - return LeapMessage(_all[-1]) - def get_all(self): """ Get all message documents for the selected mailbox. @@ -1096,11 +1098,25 @@ class MessageCollection(WithMsgFields, IndexedDB): all_docs = [doc for doc in self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_IDX, self.TYPE_MESSAGE_VAL, self.mbox)] - #if not self.is_deleted(doc)] # highly inneficient, but first let's grok it and then # let's worry about efficiency. + + # XXX FIXINDEX return sorted(all_docs, key=lambda item: item.content['uid']) + def count(self): + """ + Return the count of messages for this mailbox. + + :rtype: int + """ + count = self._soledad.get_count_from_index( + SoledadBackedAccount.TYPE_MBOX_IDX, + self.TYPE_MESSAGE_VAL, self.mbox) + return count + + # unseen messages + def unseen_iter(self): """ Get an iterator for the message docs with no `seen` flag @@ -1110,8 +1126,20 @@ class MessageCollection(WithMsgFields, IndexedDB): """ return (doc for doc in self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_SEEN_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '1', '0')) + SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, + self.TYPE_MESSAGE_VAL, self.mbox, '0')) + + def count_unseen(self): + """ + Count all messages with the `Unseen` flag. + + :returns: count + :rtype: int + """ + count = self._soledad.get_count_from_index( + SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, + self.TYPE_MESSAGE_VAL, self.mbox, '0') + return count def get_unseen(self): """ @@ -1122,6 +1150,8 @@ class MessageCollection(WithMsgFields, IndexedDB): """ return [LeapMessage(doc) for doc in self.unseen_iter()] + # recent messages + def recent_iter(self): """ Get an iterator for the message docs with `recent` flag. @@ -1143,13 +1173,17 @@ class MessageCollection(WithMsgFields, IndexedDB): """ return [LeapMessage(doc) for doc in self.recent_iter()] - def count(self): + def count_recent(self): """ - Return the count of messages for this mailbox. + Count all messages with the `Recent` flag. + :returns: count :rtype: int """ - return len(self.get_all()) + count = self._soledad.get_count_from_index( + SoledadBackedAccount.TYPE_MBOX_RECT_IDX, + self.TYPE_MESSAGE_VAL, self.mbox, '1') + return count def __len__(self): """ @@ -1179,8 +1213,7 @@ class MessageCollection(WithMsgFields, IndexedDB): :return: LeapMessage or None if not found. :rtype: LeapMessage """ - #try: - #return self.get_msg_by_uid(uid) + # XXX FIXME inneficcient, we are evaulating. try: return [doc for doc in self.get_all()][uid - 1] @@ -1252,7 +1285,7 @@ class SoledadMailbox(WithMsgFields): self._soledad = soledad self.messages = MessageCollection( - mbox=mbox, soledad=soledad) + mbox=mbox, soledad=self._soledad) if not self.getFlags(): self.setFlags(self.INIT_FLAGS) @@ -1367,6 +1400,32 @@ class SoledadMailbox(WithMsgFields): closed = property( _get_closed, _set_closed, doc="Closed attribute.") + def _get_last_uid(self): + """ + Return the last uid for this mailbox. + + :return: the last uid for messages in this mailbox + :rtype: bool + """ + mbox = self._get_mbox() + return mbox.content.get(self.LAST_UID_KEY, 1) + + def _set_last_uid(self, uid): + """ + Sets the last uid for this mailbox. + + :param uid: the uid to be set + :type uid: int + """ + leap_assert(isinstance(uid, int), "uid has to be int") + mbox = self._get_mbox() + key = self.LAST_UID_KEY + mbox.content[key] = uid + self._soledad.put_doc(mbox) + + last_uid = property( + _get_last_uid, _set_last_uid, doc="Last_UID attribute.") + def getUIDValidity(self): """ Return the unique validity identifier for this mailbox. @@ -1396,17 +1455,18 @@ class SoledadMailbox(WithMsgFields): def getUIDNext(self): """ Return the likely UID for the next message added to this - mailbox. Currently it returns the current length incremented - by one. + mailbox. Currently it returns the higher UID incremented by + one. + + We increment the next uid *each* time this function gets called. + In this way, there will be gaps if the message with the allocated + uid cannot be saved. But that is preferable to having race conditions + if we get to parallel message adding. :rtype: int """ - last = self.messages.get_last() - if last: - nextuid = last.getUID() + 1 - else: - nextuid = 1 - return nextuid + self.last_uid += 1 + return self.last_uid def getMessageCount(self): """ @@ -1423,7 +1483,7 @@ class SoledadMailbox(WithMsgFields): :return: count of messages flagged `unseen` :rtype: int """ - return len(self.messages.get_unseen()) + return self.messages.count_unseen() def getRecentCount(self): """ @@ -1432,7 +1492,7 @@ class SoledadMailbox(WithMsgFields): :return: count of messages flagged `recent` :rtype: int """ - return len(self.messages.get_recent()) + return self.messages.count_recent() def isWriteable(self): """ @@ -1489,6 +1549,7 @@ class SoledadMailbox(WithMsgFields): """ # XXX we should treat the message as an IMessage from here uid_next = self.getUIDNext() + logger.debug('Adding msg with UID :%s' % uid_next) if flags is None: flags = tuple() else: @@ -1497,8 +1558,11 @@ class SoledadMailbox(WithMsgFields): self.messages.add_msg(message, flags=flags, date=date, uid=uid_next) - exists = len(self.messages) - recent = len(self.messages.get_recent()) + exists = self.getMessageCount() + recent = self.getRecentCount() + logger.debug("there are %s messages, %s recent" % ( + exists, + recent)) for listener in self.listeners: listener.newMessages(exists, recent) return defer.succeed(None) @@ -1564,12 +1628,7 @@ class SoledadMailbox(WithMsgFields): iter(messages) except TypeError: # looks like we cannot iterate - last = self.messages.get_last() - if last is None: - uid_last = 1 - else: - uid_last = last.getUID() - messages.last = uid_last + messages.last = self.last_uid # for sequence numbers (uid = 0) if sequence: @@ -1588,14 +1647,37 @@ class SoledadMailbox(WithMsgFields): else: print "fetch %s, no msg found!!!" % msg_id + if self.isWriteable(): + self._unset_recent_flag() return tuple(result) + def _unset_recent_flag(self): + """ + Unsets `Recent` flag from a tuple of messages. + Called from fetch. + + From RFC, about `Recent`: + + Message is "recently" arrived in this mailbox. This session + is the first session to have been notified about this + message; if the session is read-write, subsequent sessions + will not see \Recent set for this message. This flag can not + be altered by the client. + + If it is not possible to determine whether or not this + session is the first session to be notified about a message, + then that message SHOULD be considered recent. + """ + for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()): + newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,)) + self._update(newflags) + def _signal_unread_to_ui(self): """ Sends unread event to ui. """ - leap_events.signal( - IMAP_UNREAD_MAIL, str(self.getUnseenCount())) + unseen = self.getUnseenCount() + leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) def store(self, messages, flags, mode, uid): """ @@ -1627,6 +1709,10 @@ class SoledadMailbox(WithMsgFields): read-write. """ # XXX implement also sequence (uid = 0) + # XXX we should prevent cclient from setting Recent flag. + leap_assert(not isinstance(flags, basestring), + "flags cannot be a string") + flags = tuple(flags) if not self.isWriteable(): log.msg('read only mailbox!') -- cgit v1.2.3 From 0de2307c11338bf4c5e36dd9fe76f445b700c288 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 19 Dec 2013 13:30:10 -0400 Subject: deferToThread unsetting recent flag --- src/leap/mail/imap/server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index b79e691..c79cf85 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -31,8 +31,10 @@ from zope.proxy import sameProxiedObjects from twisted.mail import imap4 from twisted.internet import defer +from twisted.internet.threads import deferToThread from twisted.python import log + from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type @@ -1648,7 +1650,8 @@ class SoledadMailbox(WithMsgFields): print "fetch %s, no msg found!!!" % msg_id if self.isWriteable(): - self._unset_recent_flag() + deferToThread(self._unset_recent_flag) + return tuple(result) def _unset_recent_flag(self): @@ -1668,6 +1671,7 @@ class SoledadMailbox(WithMsgFields): session is the first session to be notified about a message, then that message SHOULD be considered recent. """ + log.msg('unsetting recent flags...') for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()): newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,)) self._update(newflags) -- cgit v1.2.3 From 9460ea1bb8fd7e536aa3dcf3ed746e3765c96fa1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 20 Dec 2013 13:41:34 -0400 Subject: use soledad_writer for puts also --- src/leap/mail/imap/server.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index c79cf85..f77bf2c 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -755,7 +755,7 @@ class LeapMessage(WithMsgFields): logger.error("Unicode error {0}".format(e)) content = content.encode(charset, 'replace') fd.write(content) - # SHOULD use a separate BODY FIELD ... + # XXX SHOULD use a separate BODY FIELD ... fd.seek(0) return fd @@ -856,7 +856,12 @@ class SoledadDocWriter(object): empty = queue.empty() while not empty: item = queue.get() - self._soledad.create_doc(item) + payload = item['payload'] + mode = item['mode'] + if mode == "create": + self._soledad.create_doc(payload) + elif mode == "put": + self._soledad.put_doc(payload) empty = queue.empty() @@ -925,7 +930,7 @@ class MessageCollection(WithMsgFields, IndexedDB): # to be processed serially by the consumer (the writer). We just # need to `put` the new material on its plate. - self._soledad_writer = MessageProducer( + self.soledad_writer = MessageProducer( SoledadDocWriter(soledad), period=0.1) @@ -1003,7 +1008,10 @@ class MessageCollection(WithMsgFields, IndexedDB): content[self.UID_KEY] = uid logger.debug('enqueuing message for write') - self._soledad_writer.put(content) + + # XXX create namedtuple + self.soledad_writer.put({"mode": "create", + "payload": content}) # XXX have to decide what shall we do with errors with this change... #return self._soledad.create_doc(content) @@ -1650,7 +1658,7 @@ class SoledadMailbox(WithMsgFields): print "fetch %s, no msg found!!!" % msg_id if self.isWriteable(): - deferToThread(self._unset_recent_flag) + self._unset_recent_flag() return tuple(result) @@ -1761,8 +1769,9 @@ class SoledadMailbox(WithMsgFields): """ Updates document in u1db database """ - #log.msg('updating doc... %s ' % doc) - self._soledad.put_doc(doc) + # XXX create namedtuple + self.messages.soledad_writer.put({"mode": "put", + "payload": doc}) def __repr__(self): """ -- cgit v1.2.3 From ee1fa7da3bdc2de2bd12c55a4da9ccc291d3e82c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 20 Dec 2013 16:36:21 -0400 Subject: safety catch against wrong last_uid --- src/leap/mail/imap/server.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index f77bf2c..d92ab9d 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -1430,7 +1430,22 @@ class SoledadMailbox(WithMsgFields): leap_assert(isinstance(uid, int), "uid has to be int") mbox = self._get_mbox() key = self.LAST_UID_KEY - mbox.content[key] = uid + + count = mbox.getMessageCount() + + # XXX safety-catch. If we do get duplicates, + # we want to avoid further duplication. + + if uid >= count: + value = uid + else: + # something is wrong, + # just set the last uid + # beyond the max msg count. + logger.debug("WRONG uid < count. Setting last uid to ", count) + value = count + + mbox.content[key] = value self._soledad.put_doc(mbox) last_uid = property( -- cgit v1.2.3 From aaedde4a6a04a77e06a332e1ab917afd60f72205 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 20 Dec 2013 17:06:57 -0400 Subject: fix wrong object call --- src/leap/mail/imap/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index d92ab9d..5672e25 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -1431,7 +1431,7 @@ class SoledadMailbox(WithMsgFields): mbox = self._get_mbox() key = self.LAST_UID_KEY - count = mbox.getMessageCount() + count = self.getMessageCount() # XXX safety-catch. If we do get duplicates, # we want to avoid further duplication. -- cgit v1.2.3 From 8d4e17a279218de99b495955e96672587cb237e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Fri, 20 Dec 2013 19:20:50 -0300 Subject: Limit the size of the returned messages from IMAP to MUA to 100 --- src/leap/mail/imap/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 5672e25..2739f8c 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -1675,7 +1675,7 @@ class SoledadMailbox(WithMsgFields): if self.isWriteable(): self._unset_recent_flag() - return tuple(result) + return tuple(result[:100]) def _unset_recent_flag(self): """ -- cgit v1.2.3 From 6684735dc24ea02649b55e5fd795c7d2f5824d34 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 24 Dec 2013 09:27:43 -0200 Subject: Fix parsing of IMAP folder names (#4830). --- src/leap/mail/imap/server.py | 67 ++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 28 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 2739f8c..b9b72d0 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -22,6 +22,7 @@ import logging import StringIO import cStringIO import time +import re from collections import defaultdict from email.parser import Parser @@ -205,6 +206,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): WithMsgFields.LAST_UID_KEY: 0 } + INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + def __init__(self, account_name, soledad=None): """ Creates a SoledadAccountIndex that keeps track of the mailboxes @@ -222,7 +225,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): # XXX SHOULD assert too that the name matches the user/uuid with which # soledad has been initialized. - self._account_name = account_name.upper() + self._account_name = self._parse_mailbox_name(account_name) self._soledad = soledad self.initialize_db() @@ -241,19 +244,30 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): """ return copy.deepcopy(self.EMPTY_MBOX) + def _parse_mailbox_name(self, name): + """ + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + if self.INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return self.INBOX_NAME + name[len(self.INBOX_NAME):] + return name + def _get_mailbox_by_name(self, name): """ - Returns an mbox document by name. + Return an mbox document by name. :param name: the name of the mailbox :type name: str :rtype: SoledadDocument """ - # XXX only upper for INBOX --- - name = name.upper() doc = self._soledad.get_from_index( - self.TYPE_MBOX_IDX, self.MBOX_KEY, name) + self.TYPE_MBOX_IDX, self.MBOX_KEY, + self._parse_mailbox_name(name)) return doc[0] if doc else None @property @@ -261,7 +275,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): """ A list of the current mailboxes for this account. """ - return [str(doc.content[self.MBOX_KEY]) + return [doc.content[self.MBOX_KEY] for doc in self._soledad.get_from_index( self.TYPE_IDX, self.MBOX_KEY)] @@ -270,7 +284,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): """ A list of the current subscriptions for this account. """ - return [str(doc.content[self.MBOX_KEY]) + return [doc.content[self.MBOX_KEY] for doc in self._soledad.get_from_index( self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] @@ -284,8 +298,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :returns: a a SoledadMailbox instance :rtype: SoledadMailbox """ - # XXX only upper for INBOX - name = name.upper() + name = self._parse_mailbox_name(name) + if name not in self.mailboxes: raise imap4.MailboxException("No such mailbox") @@ -297,12 +311,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): def addMailbox(self, name, creation_ts=None): """ - Adds a mailbox to the account. + Add a mailbox to the account. :param name: the name of the mailbox :type name: str - :param creation_ts: a optional creation timestamp to be used as + :param creation_ts: an optional creation timestamp to be used as mailbox id. A timestamp will be used if no one is provided. :type creation_ts: int @@ -310,9 +324,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :returns: True if successful :rtype: bool """ - # XXX only upper for INBOX - name = name.upper() - # XXX should check mailbox name for RFC-compliant form + name = self._parse_mailbox_name(name) if name in self.mailboxes: raise imap4.MailboxCollision, name @@ -321,7 +333,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): # by default, we pass an int value # taken from the current time # we make sure to take enough decimals to get a unique - # maibox-uidvalidity. + # mailbox-uidvalidity. creation_ts = int(time.time() * 10E2) mbox = self._get_empty_mailbox() @@ -346,8 +358,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :raise MailboxException: Raised if this mailbox cannot be added. """ # TODO raise MailboxException - - paths = filter(None, pathspec.split('/')) + paths = filter(None, + self._parse_mailbox_name(pathspec).split('/')) for accum in range(1, len(paths)): try: self.addMailbox('/'.join(paths[:accum])) @@ -372,13 +384,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :rtype: bool """ - # XXX only upper for INBOX - name = name.upper() + name = self._parse_mailbox_name(name) if name not in self.mailboxes: return None - self.selected = str(name) + self.selected = name return SoledadMailbox( name, rw=readwrite, @@ -398,8 +409,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): names. use with care. :type force: bool """ - # XXX only upper for INBOX - name = name.upper() + name = self._parse_mailbox_name(name) + if not name in self.mailboxes: raise imap4.MailboxException("No such mailbox") @@ -436,9 +447,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :param newname: new name of the mailbox :type newname: str """ - # XXX only upper for INBOX - oldname = oldname.upper() - newname = newname.upper() + oldname = self._parse_mailbox_name(oldname) + newname = self._parse_mailbox_name(newname) if oldname not in self.mailboxes: raise imap4.NoSuchMailbox, oldname @@ -516,7 +526,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :param name: name of the mailbox :type name: str """ - name = name.upper() + name = self._parse_mailbox_name(name) if name not in self.subscriptions: self._set_subscription(name, True) @@ -527,7 +537,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :param name: name of the mailbox :type name: str """ - name = name.upper() + name = self._parse_mailbox_name(name) if name not in self.subscriptions: raise imap4.MailboxException, "Not currently subscribed to " + name self._set_subscription(name, False) @@ -549,7 +559,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :type wildcard: str """ # XXX use wildcard in index query - ref = self._inferiorNames(ref.upper()) + ref = self._inferiorNames( + self._parse_mailbox_name(ref)) wildcard = imap4.wildcardToRegexp(wildcard, '/') return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] -- cgit v1.2.3 From a65aba63cf90efcf036ed7ba0d515865c5e8457e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 24 Dec 2013 20:28:58 -0400 Subject: defer costly operations --- src/leap/mail/imap/server.py | 88 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index b9b72d0..e97ed2a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -21,11 +21,13 @@ import copy import logging import StringIO import cStringIO +import os import time import re from collections import defaultdict from email.parser import Parser +from functools import wraps from zope.interface import implements from zope.proxy import sameProxiedObjects @@ -35,6 +37,7 @@ from twisted.internet import defer from twisted.internet.threads import deferToThread from twisted.python import log +from u1db import errors as u1db_errors from leap.common import events as leap_events from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL @@ -46,6 +49,65 @@ from leap.soledad.client import Soledad logger = logging.getLogger(__name__) +def deferred(f): + ''' + Decorator, for deferring methods to Threads. + + It will do a deferToThread of the decorated method + unless the environment variable LEAPMAIL_DEBUG is set. + + It uses a descriptor to delay the definition of the + method wrapper. + ''' + class descript(object): + def __init__(self, f): + self.f = f + + def __get__(self, instance, klass): + if instance is None: + # Class method was requested + return self.make_unbound(klass) + return self.make_bound(instance) + + def _errback(self, failure): + err = failure.value + #logger.error(err) + log.err(err) + + def make_unbound(self, klass): + + @wraps(self.f) + def wrapper(*args, **kwargs): + '''This documentation will vanish :)''' + raise TypeError( + 'unbound method {}() must be called with {} instance ' + 'as first argument (got nothing instead)'.format( + self.f.__name__, + klass.__name__) + ) + return wrapper + + def make_bound(self, instance): + + @wraps(self.f) + def wrapper(*args, **kwargs): + '''This documentation will disapear :)''' + + if not os.environ.get('LEAPMAIL_DEBUG'): + d = deferToThread(self.f, instance, *args, **kwargs) + d.addErrback(self._errback) + return d + else: + return self.f(instance, *args, **kwargs) + + # This instance does not need the descriptor anymore, + # let it find the wrapper directly next time: + setattr(instance, self.f.__name__, wrapper) + return wrapper + + return descript(f) + + class MissingIndexError(Exception): """ Raises when tried to access a non existent index document. @@ -870,9 +932,19 @@ class SoledadDocWriter(object): payload = item['payload'] mode = item['mode'] if mode == "create": - self._soledad.create_doc(payload) + call = self._soledad.create_doc elif mode == "put": - self._soledad.put_doc(payload) + call = self._soledad.put_doc + + # should handle errors + try: + call(payload) + except u1db_errors.RevisionConflict as exc: + logger.error("Error: %r" % (exc,)) + # XXX DEBUG -- remove-me + #logger.debug("conflicting doc: %s" % payload) + raise exc + empty = queue.empty() @@ -954,6 +1026,7 @@ class MessageCollection(WithMsgFields, IndexedDB): """ return copy.deepcopy(self.EMPTY_MSG) + @deferred def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): """ Creates a new message document. @@ -1639,6 +1712,7 @@ class SoledadMailbox(WithMsgFields): # more generically return [x for x in range(len(deleted))] + @deferred def fetch(self, messages, uid): """ Retrieve one or more messages in this mailbox. @@ -1668,6 +1742,7 @@ class SoledadMailbox(WithMsgFields): # for sequence numbers (uid = 0) if sequence: + logger.debug("Getting msg by index: INEFFICIENT call!") for msg_id in messages: msg = self.messages.get_msg_by_index(msg_id - 1) if msg: @@ -1686,8 +1761,11 @@ class SoledadMailbox(WithMsgFields): if self.isWriteable(): self._unset_recent_flag() - return tuple(result[:100]) + # XXX workaround for hangs in thunderbird + #return tuple(result[:100]) + return tuple(result) + @deferred def _unset_recent_flag(self): """ Unsets `Recent` flag from a tuple of messages. @@ -1706,10 +1784,12 @@ class SoledadMailbox(WithMsgFields): then that message SHOULD be considered recent. """ log.msg('unsetting recent flags...') + for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()): newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,)) self._update(newflags) + @deferred def _signal_unread_to_ui(self): """ Sends unread event to ui. @@ -1717,6 +1797,7 @@ class SoledadMailbox(WithMsgFields): unseen = self.getUnseenCount() leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) + @deferred def store(self, messages, flags, mode, uid): """ Sets the flags of one or more messages. @@ -1774,6 +1855,7 @@ class SoledadMailbox(WithMsgFields): self._signal_unread_to_ui() return result + @deferred def close(self): """ Expunge and mark as closed -- cgit v1.2.3 From 70fdb37740b0737202ad08b4637bf23c12dab87e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 25 Dec 2013 02:46:27 -0400 Subject: Move flags and other metadata to a separate doc. This change will allow for quicker access times, and smaller syncs since the fields that change more often will fall in a pretty small document. For the big raw message, we only need to sync once. Also, implemented multipart interface for messages. This will need additional migration helper in --repair-mailboxes. --- src/leap/mail/imap/server.py | 612 ++++++++++++++++++++++++------------------- 1 file changed, 338 insertions(+), 274 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index e97ed2a..8758dcb 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -25,7 +25,7 @@ import os import time import re -from collections import defaultdict +from collections import defaultdict, namedtuple from email.parser import Parser from functools import wraps @@ -71,7 +71,7 @@ def deferred(f): def _errback(self, failure): err = failure.value - #logger.error(err) + logger.warning('error in method: %s' % (self.f.__name__)) log.err(err) def make_unbound(self, klass): @@ -133,6 +133,8 @@ class WithMsgFields(object): RAW_KEY = "raw" SUBJECT_KEY = "subject" UID_KEY = "uid" + MULTIPART_KEY = "multi" + SIZE_KEY = "size" # Mailbox specific keys CLOSED_KEY = "closed" @@ -145,6 +147,8 @@ class WithMsgFields(object): TYPE_KEY = "type" TYPE_MESSAGE_VAL = "msg" TYPE_MBOX_VAL = "mbox" + TYPE_FLAGS_VAL = "flags" + # should add also a headers val INBOX_VAL = "inbox" @@ -166,6 +170,8 @@ class WithMsgFields(object): SUBJECT_FIELD = "Subject" DATE_FIELD = "Date" +fields = WithMsgFields # alias for convenience + class IndexedDB(object): """ @@ -209,12 +215,79 @@ class IndexedDB(object): self._soledad.create_index(name, *expression) +class MailParser(object): + """ + Mixin with utility methods to parse raw messages. + """ + def __init__(self): + """ + Initializes the mail parser. + """ + self._parser = Parser() + + def _get_parsed_msg(self, raw): + """ + Return a parsed Message. + + :param raw: the raw string to parse + :type raw: basestring, or StringIO object + """ + msg = self._get_parser_fun(raw)(raw, True) + return msg + + def _get_parser_fun(self, o): + """ + Retunn the proper parser function for an object. + + :param o: object + :type o: object + :param parser: an instance of email.parser.Parser + :type parser: email.parser.Parser + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return self._parser.parse + if isinstance(o, basestring): + return self._parser.parsestr + + def _stringify(self, o): + """ + Return a string object. + + :param o: object + :type o: object + """ + if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): + return o.getvalue() + else: + return o + + +class MBoxParser(object): + """ + Utility function to parse mailbox names. + """ + INBOX_NAME = "INBOX" + INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) + + def _parse_mailbox_name(self, name): + """ + :param name: the name of the mailbox + :type name: unicode + + :rtype: unicode + """ + if self.INBOX_RE.match(name): + # ensure inital INBOX is uppercase + return self.INBOX_NAME + name[len(self.INBOX_NAME):] + return name + + ####################################### # Soledad Account ####################################### -class SoledadBackedAccount(WithMsgFields, IndexedDB): +class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): """ An implementation of IAccount and INamespacePresenteer that is backed by Soledad Encrypted Documents. @@ -254,12 +327,11 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): 'bool(recent)', 'bool(seen)'], } - INBOX_NAME = "INBOX" MBOX_KEY = MBOX_VAL EMPTY_MBOX = { WithMsgFields.TYPE_KEY: MBOX_KEY, - WithMsgFields.TYPE_MBOX_VAL: INBOX_NAME, + WithMsgFields.TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, WithMsgFields.SUBJECT_KEY: "", WithMsgFields.FLAGS_KEY: [], WithMsgFields.CLOSED_KEY: False, @@ -268,8 +340,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): WithMsgFields.LAST_UID_KEY: 0 } - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - def __init__(self, account_name, soledad=None): """ Creates a SoledadAccountIndex that keeps track of the mailboxes @@ -306,18 +376,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): """ return copy.deepcopy(self.EMPTY_MBOX) - def _parse_mailbox_name(self, name): - """ - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name - def _get_mailbox_by_name(self, name): """ Return an mbox document by name. @@ -420,7 +478,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): :raise MailboxException: Raised if this mailbox cannot be added. """ # TODO raise MailboxException - paths = filter(None, + paths = filter( + None, self._parse_mailbox_name(pathspec).split('/')) for accum in range(1, len(paths)): try: @@ -665,19 +724,43 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB): ####################################### -class LeapMessage(WithMsgFields): +class LeapMessage(fields, MailParser, MBoxParser): - implements(imap4.IMessage, imap4.IMessageFile) + implements(imap4.IMessage) - def __init__(self, doc): + def __init__(self, soledad, uid, mbox): """ Initializes a LeapMessage. - :param doc: A SoledadDocument containing the internal - representation of the message - :type doc: SoledadDocument + :param soledad: a Soledad instance + :type soledad: Soledad + :param uid: the UID for the message. + :type uid: int or basestring + :param mbox: the mbox this message belongs to + :type mbox: basestring """ - self._doc = doc + MailParser.__init__(self) + self._soledad = soledad + self._uid = int(uid) + self._mbox = self._parse_mailbox_name(mbox) + + self.__cdoc = None + + @property + def _fdoc(self): + """ + An accessor to the flags docuemnt + """ + return self._get_flags_doc() + + @property + def _cdoc(self): + """ + An accessor to the content docuemnt + """ + if not self.__cdoc: + self.__cdoc = self._get_content_doc() + return self.__cdoc def getUID(self): """ @@ -686,11 +769,7 @@ class LeapMessage(WithMsgFields): :return: uid for this message :rtype: int """ - # XXX debug, to remove after a while... - if not self._doc: - log.msg('BUG!!! ---- message has no doc!') - return - return self._doc.content[self.UID_KEY] + return self._uid def getFlags(self): """ @@ -699,9 +778,13 @@ class LeapMessage(WithMsgFields): :return: The flags, represented as strings :rtype: tuple """ - if self._doc is None: + if self._uid is None: return [] - flags = self._doc.content.get(self.FLAGS_KEY, None) + + flags = [] + flag_doc = self._fdoc + if flag_doc: + flags = flag_doc.content.get(self.FLAGS_KEY, None) if flags: flags = map(str, flags) return tuple(flags) @@ -722,12 +805,13 @@ class LeapMessage(WithMsgFields): :rtype: SoledadDocument """ leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - log.msg('setting flags') - doc = self._doc + log.msg('setting flags: %s' % (self._uid)) + + doc = self._fdoc doc.content[self.FLAGS_KEY] = flags doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags - return doc + self._soledad.put_doc(doc) def addFlags(self, flags): """ @@ -743,7 +827,7 @@ class LeapMessage(WithMsgFields): """ leap_assert(isinstance(flags, tuple), "flags need to be a tuple") oldflags = self.getFlags() - return self.setFlags(tuple(set(flags + oldflags))) + self.setFlags(tuple(set(flags + oldflags))) def removeFlags(self, flags): """ @@ -759,7 +843,7 @@ class LeapMessage(WithMsgFields): """ leap_assert(isinstance(flags, tuple), "flags need to be a tuple") oldflags = self.getFlags() - return self.setFlags(tuple(set(oldflags) - set(flags))) + self.setFlags(tuple(set(oldflags) - set(flags))) def getInternalDate(self): """ @@ -768,48 +852,14 @@ class LeapMessage(WithMsgFields): :rtype: C{str} :return: An RFC822-formatted date string. """ - return str(self._doc.content.get(self.DATE_KEY, '')) - - # - # IMessageFile - # - - """ - Optional message interface for representing messages as files. - - If provided by message objects, this interface will be used instead - the more complex MIME-based interface. - """ - - def open(self): - """ - Return an file-like object opened for reading. - - Reading from the returned file will return all the bytes - of which this message consists. - - :return: file-like object opened fore reading. - :rtype: StringIO - """ - fd = cStringIO.StringIO() - content = self._doc.content.get(self.RAW_KEY, '') - charset = get_email_charset( - unicode(self._doc.content.get(self.RAW_KEY, ''))) - try: - content = content.encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - content = content.encode(charset, 'replace') - fd.write(content) - fd.seek(0) - return fd + return str(self._cdoc.content.get(self.DATE_KEY, '')) # # IMessagePart # - # XXX should implement the rest of IMessagePart interface: - # (and do not use the open above) + # XXX we should implement this interface too for the subparts + # so we allow nested parts... def getBodyFile(self): """ @@ -819,15 +869,21 @@ class LeapMessage(WithMsgFields): :rtype: StringIO """ fd = StringIO.StringIO() - content = self._doc.content.get(self.RAW_KEY, '') + + cdoc = self._cdoc + content = cdoc.content.get(self.RAW_KEY, '') charset = get_email_charset( - unicode(self._doc.content.get(self.RAW_KEY, ''))) + unicode(cdoc.content.get(self.RAW_KEY, ''))) try: content = content.encode(charset) except (UnicodeEncodeError, UnicodeDecodeError) as e: logger.error("Unicode error {0}".format(e)) content = content.encode(charset, 'replace') - fd.write(content) + + raw = self._get_raw_msg() + msg = self._get_parsed_msg(raw) + body = msg.get_payload() + fd.write(body) # XXX SHOULD use a separate BODY FIELD ... fd.seek(0) return fd @@ -839,13 +895,18 @@ class LeapMessage(WithMsgFields): :return: size of the message, in octets :rtype: int """ - return self.getBodyFile().len + size = self._cdoc.content.get(self.SIZE_KEY, False) + if not size: + # XXX fallback, should remove when all migrated. + size = self.getBodyFile().len + return size def _get_headers(self): """ Return the headers dict stored in this message document. """ - return self._doc.content.get(self.HEADERS_KEY, {}) + # XXX get from the headers doc + return self._cdoc.content.get(self.HEADERS_KEY, {}) def getHeaders(self, negate, *names): """ @@ -876,30 +937,90 @@ class LeapMessage(WithMsgFields): if cond(key)] return dict(filter_by_cond) - # --- no multipart for now - # XXX Fix MULTIPART SUPPORT! - def isMultipart(self): - return False + """ + Return True if this message is multipart. + """ + if self._cdoc: + retval = self._cdoc.content.get(self.MULTIPART_KEY, False) + print "MULTIPART? ", retval - def getSubPart(part): - return None + def getSubPart(self, part): + """ + Retrieve a MIME submessage + + :type part: C{int} + :param part: The number of the part to retrieve, indexed from 0. + :raise IndexError: Raised if the specified part does not exist. + :raise TypeError: Raised if this message is not multipart. + :rtype: Any object implementing C{IMessagePart}. + :return: The specified sub-part. + """ + if not self.isMultipart(): + raise TypeError + + msg = self._get_parsed_msg() + # XXX should wrap IMessagePart + return msg.get_payload()[part] # # accessors # + def _get_flags_doc(self): + """ + Return the document that keeps the flags for this + message. + """ + flag_docs = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_UID_IDX, + fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) + flag_doc = flag_docs[0] if flag_docs else None + return flag_doc + + def _get_content_doc(self): + """ + Return the document that keeps the flags for this + message. + """ + cont_docs = self._soledad.get_from_index( + SoledadBackedAccount.TYPE_MBOX_UID_IDX, + fields.TYPE_MESSAGE_VAL, self._mbox, str(self._uid)) + cont_doc = cont_docs[0] if cont_docs else None + return cont_doc + + def _get_raw_msg(self): + """ + Return the raw msg. + :rtype: basestring + """ + return self._cdoc.content.get(self.RAW_KEY, '') + def __getitem__(self, key): """ Return the content of the message document. - @param key: The key - @type key: str + :param key: The key + :type key: str - @return: The content value indexed by C{key} or None - @rtype: str + :return: The content value indexed by C{key} or None + :rtype: str """ - return self._doc.content.get(key, None) + return self._cdoc.content.get(key, None) + + def does_exist(self): + """ + Return True if there is actually a message for this + UID and mbox. + """ + return bool(self._fdoc) + + +SoledadWriterPayload = namedtuple( + 'SoledadWriterPayload', ['mode', 'payload']) + +SoledadWriterPayload.CREATE = 1 +SoledadWriterPayload.PUT = 2 class SoledadDocWriter(object): @@ -929,26 +1050,22 @@ class SoledadDocWriter(object): empty = queue.empty() while not empty: item = queue.get() - payload = item['payload'] - mode = item['mode'] - if mode == "create": + if item.mode == SoledadWriterPayload.CREATE: call = self._soledad.create_doc - elif mode == "put": + elif item.mode == SoledadWriterPayload.PUT: call = self._soledad.put_doc # should handle errors try: - call(payload) + call(item.payload) except u1db_errors.RevisionConflict as exc: logger.error("Error: %r" % (exc,)) - # XXX DEBUG -- remove-me - #logger.debug("conflicting doc: %s" % payload) raise exc empty = queue.empty() -class MessageCollection(WithMsgFields, IndexedDB): +class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): """ A collection of messages, surprisingly. @@ -959,16 +1076,27 @@ class MessageCollection(WithMsgFields, IndexedDB): # XXX this should be able to produce a MessageSet methinks EMPTY_MSG = { - WithMsgFields.TYPE_KEY: WithMsgFields.TYPE_MESSAGE_VAL, - WithMsgFields.UID_KEY: 1, - WithMsgFields.MBOX_KEY: WithMsgFields.INBOX_VAL, - WithMsgFields.SUBJECT_KEY: "", - WithMsgFields.DATE_KEY: "", - WithMsgFields.SEEN_KEY: False, - WithMsgFields.RECENT_KEY: True, - WithMsgFields.FLAGS_KEY: [], - WithMsgFields.HEADERS_KEY: {}, - WithMsgFields.RAW_KEY: "", + fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, + fields.UID_KEY: 1, + fields.MBOX_KEY: fields.INBOX_VAL, + + fields.SUBJECT_KEY: "", + fields.DATE_KEY: "", + fields.RAW_KEY: "", + + # XXX should separate headers into another doc + fields.HEADERS_KEY: {}, + } + + EMPTY_FLAGS = { + fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, + fields.UID_KEY: 1, + fields.MBOX_KEY: fields.INBOX_VAL, + + fields.FLAGS_KEY: [], + fields.SEEN_KEY: False, + fields.RECENT_KEY: True, + fields.MULTIPART_KEY: False, } # get from SoledadBackedAccount the needed index-related constants @@ -987,25 +1115,17 @@ class MessageCollection(WithMsgFields, IndexedDB): :param soledad: Soledad database :type soledad: Soledad instance """ - # XXX pass soledad directly - + MailParser.__init__(self) leap_assert(mbox, "Need a mailbox name to initialize") leap_assert(mbox.strip() != "", "mbox cannot be blank space") leap_assert(isinstance(mbox, (str, unicode)), "mbox needs to be a string") leap_assert(soledad, "Need a soledad instance to initialize") - # This is a wrapper now!... - # should move assertion there... - #leap_assert(isinstance(soledad._db, SQLCipherDatabase), - #"soledad._db must be an instance of SQLCipherDatabase") - # okay, all in order, keep going... - - self.mbox = mbox.upper() + self.mbox = self._parse_mailbox_name(mbox) self._soledad = soledad self.initialize_db() - self._parser = Parser() # I think of someone like nietzsche when reading this @@ -1015,7 +1135,7 @@ class MessageCollection(WithMsgFields, IndexedDB): self.soledad_writer = MessageProducer( SoledadDocWriter(soledad), - period=0.1) + period=0.05) def _get_empty_msg(self): """ @@ -1026,6 +1146,15 @@ class MessageCollection(WithMsgFields, IndexedDB): """ return copy.deepcopy(self.EMPTY_MSG) + def _get_empty_flags_doc(self): + """ + Returns an empty doc for storing flags. + + :return: + :rtype: + """ + return copy.deepcopy(self.EMPTY_FLAGS) + @deferred def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): """ @@ -1046,58 +1175,57 @@ class MessageCollection(WithMsgFields, IndexedDB): :param uid: the message uid for this mailbox :type uid: int """ + # TODO: split in smaller methods logger.debug('adding message') if flags is None: flags = tuple() leap_assert_type(flags, tuple) - def stringify(o): - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return o.getvalue() - else: - return o + content_doc = self._get_empty_msg() + flags_doc = self._get_empty_flags_doc() - content = self._get_empty_msg() - content[self.MBOX_KEY] = self.mbox + content_doc[self.MBOX_KEY] = self.mbox + flags_doc[self.MBOX_KEY] = self.mbox + # ...should get a sanity check here. + content_doc[self.UID_KEY] = uid + flags_doc[self.UID_KEY] = uid if flags: - content[self.FLAGS_KEY] = map(stringify, flags) - content[self.SEEN_KEY] = self.SEEN_FLAG in flags + flags_doc[self.FLAGS_KEY] = map(self._stringify, flags) + flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags - def _get_parser_fun(o): - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return self._parser.parse - if isinstance(o, (str, unicode)): - return self._parser.parsestr - - msg = _get_parser_fun(raw)(raw, True) + msg = self._get_parsed_msg(raw) headers = dict(msg) + flags_doc[self.MULTIPART_KEY] = msg.is_multipart() # XXX get lower case for keys? - content[self.HEADERS_KEY] = headers + # XXX get headers doc + content_doc[self.HEADERS_KEY] = headers # set subject based on message headers and eventually replace by # subject given as param if self.SUBJECT_FIELD in headers: - content[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] + content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] if subject is not None: - content[self.SUBJECT_KEY] = subject - content[self.RAW_KEY] = stringify(raw) + content_doc[self.SUBJECT_KEY] = subject + + # XXX could separate body into its own doc + # but should also separate multiparts + # that should be wrapped in MessagePart + content_doc[self.RAW_KEY] = self._stringify(raw) + content_doc[self.SIZE_KEY] = len(raw) if not date and self.DATE_FIELD in headers: - content[self.DATE_KEY] = headers[self.DATE_FIELD] + content_doc[self.DATE_KEY] = headers[self.DATE_FIELD] else: - content[self.DATE_KEY] = date - - # ...should get a sanity check here. - content[self.UID_KEY] = uid + content_doc[self.DATE_KEY] = date logger.debug('enqueuing message for write') - # XXX create namedtuple - self.soledad_writer.put({"mode": "create", - "payload": content}) - # XXX have to decide what shall we do with errors with this change... - #return self._soledad.create_doc(content) + ptuple = SoledadWriterPayload + self.soledad_writer.put(ptuple( + mode=ptuple.CREATE, payload=content_doc)) + self.soledad_writer.put(ptuple( + mode=ptuple.CREATE, payload=flags_doc)) def remove(self, msg): """ @@ -1110,23 +1238,6 @@ class MessageCollection(WithMsgFields, IndexedDB): # getters - def get_by_uid(self, uid): - """ - Retrieves a message document by UID. - - :param uid: the message uid to query by - :type uid: int - - :return: A SoledadDocument instance matching the query, - or None if not found. - :rtype: SoledadDocument - """ - docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, str(uid)) - - return docs[0] if docs else None - def get_msg_by_uid(self, uid): """ Retrieves a LeapMessage by UID. @@ -1138,43 +1249,10 @@ class MessageCollection(WithMsgFields, IndexedDB): or None if not found. :rtype: LeapMessage """ - doc = self.get_by_uid(uid) - if doc: - return LeapMessage(doc) - - def get_by_index(self, index): - """ - Retrieves a mesage document by mailbox index. - - :param index: the index of the sequence (zero-indexed) - :type index: int - """ - # XXX inneficient! ---- we should keep an index document - # with uid -- doc_uuid :) - try: - return self.get_all()[index] - except IndexError: + msg = LeapMessage(self._soledad, uid, self.mbox) + if not msg.does_exist(): return None - - def get_msg_by_index(self, index): - """ - Retrieves a LeapMessage by sequence index. - - :param index: the index of the sequence (zero-indexed) - :type index: int - """ - doc = self.get_by_index(index) - if doc: - return LeapMessage(doc) - - def is_deleted(self, doc): - """ - Returns whether a given doc is deleted or not. - - :param doc: the document to check - :rtype: bool - """ - return self.DELETED_FLAG in doc.content[self.FLAGS_KEY] + return msg def get_all(self): """ @@ -1184,18 +1262,19 @@ class MessageCollection(WithMsgFields, IndexedDB): :return: a list of u1db documents :rtype: list of SoledadDocument """ + # TODO change to get_all_docs and turn this + # into returning messages if sameProxiedObjects(self._soledad, None): logger.warning('Tried to get messages but soledad is None!') return [] - #f XXX this should return LeapMessage instances all_docs = [doc for doc in self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MESSAGE_VAL, self.mbox)] - # highly inneficient, but first let's grok it and then - # let's worry about efficiency. + fields.TYPE_FLAGS_VAL, self.mbox)] - # XXX FIXINDEX + # inneficient, but first let's grok it and then + # let's worry about efficiency. + # XXX FIXINDEX -- should implement order by in soledad return sorted(all_docs, key=lambda item: item.content['uid']) def count(self): @@ -1206,7 +1285,7 @@ class MessageCollection(WithMsgFields, IndexedDB): """ count = self._soledad.get_count_from_index( SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MESSAGE_VAL, self.mbox) + fields.TYPE_FLAGS_VAL, self.mbox) return count # unseen messages @@ -1215,13 +1294,13 @@ class MessageCollection(WithMsgFields, IndexedDB): """ Get an iterator for the message docs with no `seen` flag - :return: iterator through unseen message docs + :return: iterator through unseen message doc UIDs :rtype: iterable """ - return (doc for doc in + return (doc.content[self.UID_KEY] for doc in self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '0')) + self.TYPE_FLAGS_VAL, self.mbox, '0')) def count_unseen(self): """ @@ -1232,7 +1311,7 @@ class MessageCollection(WithMsgFields, IndexedDB): """ count = self._soledad.get_count_from_index( SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '0') + self.TYPE_FLAGS_VAL, self.mbox, '0') return count def get_unseen(self): @@ -1242,7 +1321,8 @@ class MessageCollection(WithMsgFields, IndexedDB): :returns: a list of LeapMessages :rtype: list """ - return [LeapMessage(doc) for doc in self.unseen_iter()] + return [LeapMessage(self._soledad, docid, self.mbox) + for docid in self.unseen_iter()] # recent messages @@ -1253,10 +1333,10 @@ class MessageCollection(WithMsgFields, IndexedDB): :return: iterator through recent message docs :rtype: iterable """ - return (doc for doc in + return (doc.content[self.UID_KEY] for doc in self._soledad.get_from_index( SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '1')) + self.TYPE_FLAGS_VAL, self.mbox, '1')) def get_recent(self): """ @@ -1265,7 +1345,8 @@ class MessageCollection(WithMsgFields, IndexedDB): :returns: a list of LeapMessages :rtype: list """ - return [LeapMessage(doc) for doc in self.recent_iter()] + return [LeapMessage(self._soledad, docid, self.mbox) + for docid in self.recent_iter()] def count_recent(self): """ @@ -1276,7 +1357,7 @@ class MessageCollection(WithMsgFields, IndexedDB): """ count = self._soledad.get_count_from_index( SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_MESSAGE_VAL, self.mbox, '1') + self.TYPE_FLAGS_VAL, self.mbox, '1') return count def __len__(self): @@ -1297,23 +1378,6 @@ class MessageCollection(WithMsgFields, IndexedDB): # XXX return LeapMessage instead?! (change accordingly) return (m.content for m in self.get_all()) - def __getitem__(self, uid): - """ - Allows indexing as a list, with msg uid as the index. - - :param uid: an integer index - :type uid: int - - :return: LeapMessage or None if not found. - :rtype: LeapMessage - """ - # XXX FIXME inneficcient, we are evaulating. - try: - return [doc - for doc in self.get_all()][uid - 1] - except IndexError: - return None - def __repr__(self): """ Representation string for this object. @@ -1321,10 +1385,11 @@ class MessageCollection(WithMsgFields, IndexedDB): return u"" % ( self.mbox, self.count()) - # XXX should implement __eq__ also + # XXX should implement __eq__ also !!! --- use a hash + # of content for that, will be used for dedup. -class SoledadMailbox(WithMsgFields): +class SoledadMailbox(WithMsgFields, MBoxParser): """ A Soledad-backed IMAP mailbox. @@ -1373,7 +1438,7 @@ class SoledadMailbox(WithMsgFields): #leap_assert(isinstance(soledad._db, SQLCipherDatabase), #"soledad._db must be an instance of SQLCipherDatabase") - self.mbox = mbox + self.mbox = self._parse_mailbox_name(mbox) self.rw = rw self._soledad = soledad @@ -1440,11 +1505,6 @@ class SoledadMailbox(WithMsgFields): :returns: tuple of flags for this mailbox :rtype: tuple of str """ - #return map(str, self.INIT_FLAGS) - - # XXX CHECK against thunderbird XXX - # XXX I think this is slightly broken.. :/ - mbox = self._get_mbox() if not mbox: return None @@ -1458,7 +1518,6 @@ class SoledadMailbox(WithMsgFields): :param flags: a tuple with the flags :type flags: tuple of str """ - # TODO -- fix also getFlags leap_assert(isinstance(flags, tuple), "flags expected to be a tuple") mbox = self._get_mbox() @@ -1526,7 +1585,7 @@ class SoledadMailbox(WithMsgFields): # something is wrong, # just set the last uid # beyond the max msg count. - logger.debug("WRONG uid < count. Setting last uid to ", count) + logger.debug("WRONG uid < count. Setting last uid to %s", count) value = count mbox.content[key] = value @@ -1634,7 +1693,7 @@ class SoledadMailbox(WithMsgFields): if self.CMD_RECENT in names: r[self.CMD_RECENT] = self.getRecentCount() if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.getMessageCount() + 1 + r[self.CMD_UIDNEXT] = self.last_uid + 1 if self.CMD_UIDVALIDITY in names: r[self.CMD_UIDVALIDITY] = self.getUID() if self.CMD_UNSEEN in names: @@ -1664,17 +1723,34 @@ class SoledadMailbox(WithMsgFields): else: flags = tuple(str(flag) for flag in flags) + d = self._do_add_messages(message, flags, date, uid_next) + d.addCallback(self._notify_new) + + @deferred + def _do_add_messages(self, message, flags, date, uid_next): + """ + Calls to the messageCollection add_msg method (deferred to thread). + Invoked from addMessage. + """ self.messages.add_msg(message, flags=flags, date=date, uid=uid_next) + def _notify_new(self, *args): + """ + Notify of new messages to all the listeners. + + :param args: ignored. + """ exists = self.getMessageCount() recent = self.getRecentCount() - logger.debug("there are %s messages, %s recent" % ( + logger.debug("NOTIFY: there are %s messages, %s recent" % ( exists, recent)) - for listener in self.listeners: - listener.newMessages(exists, recent) - return defer.succeed(None) + + logger.debug("listeners: %s", str(self.listeners)) + for l in self.listeners: + logger.debug('notifying...') + l.newMessages(exists, recent) # commands, do not rename methods @@ -1743,15 +1819,11 @@ class SoledadMailbox(WithMsgFields): # for sequence numbers (uid = 0) if sequence: logger.debug("Getting msg by index: INEFFICIENT call!") - for msg_id in messages: - msg = self.messages.get_msg_by_index(msg_id - 1) - if msg: - result.append((msg.getUID(), msg)) - else: - print "fetch %s, no msg found!!!" % msg_id + raise NotImplementedError else: for msg_id in messages: + print "getting msg by uid", msg_id msg = self.messages.get_msg_by_uid(msg_id) if msg: result.append((msg_id, msg)) @@ -1760,9 +1832,10 @@ class SoledadMailbox(WithMsgFields): if self.isWriteable(): self._unset_recent_flag() + self._signal_unread_to_ui() # XXX workaround for hangs in thunderbird - #return tuple(result[:100]) + #return tuple(result[:100]) # --- doesn't show all!! return tuple(result) @deferred @@ -1784,10 +1857,9 @@ class SoledadMailbox(WithMsgFields): then that message SHOULD be considered recent. """ log.msg('unsetting recent flags...') - - for msg in (LeapMessage(doc) for doc in self.messages.recent_iter()): - newflags = msg.removeFlags((WithMsgFields.RECENT_FLAG,)) - self._update(newflags) + for msg in self.messages.get_recent(): + msg.removeFlags((fields.RECENT_FLAG,)) + self._signal_unread_to_ui() @deferred def _signal_unread_to_ui(self): @@ -1842,14 +1914,14 @@ class SoledadMailbox(WithMsgFields): result = {} for msg_id in messages: - print "MSG ID = %s" % msg_id + log.msg("MSG ID = %s" % msg_id) msg = self.messages.get_msg_by_uid(msg_id) if mode == 1: - self._update(msg.addFlags(flags)) + msg.addFlags(flags) elif mode == -1: - self._update(msg.removeFlags(flags)) + msg.removeFlags(flags) elif mode == 0: - self._update(msg.setFlags(flags)) + msg.setFlags(flags) result[msg_id] = msg.getFlags() self._signal_unread_to_ui() @@ -1873,14 +1945,6 @@ class SoledadMailbox(WithMsgFields): for doc in docs: self.messages._soledad.delete_doc(doc) - def _update(self, doc): - """ - Updates document in u1db database - """ - # XXX create namedtuple - self.messages.soledad_writer.put({"mode": "put", - "payload": doc}) - def __repr__(self): """ Representation string for this mailbox. -- cgit v1.2.3 From 59756d93f329ceee925a813bd4137c2cb34e7679 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 25 Dec 2013 11:57:42 -0400 Subject: inlineCallbacks all the things! --- src/leap/mail/imap/server.py | 78 +++++++------------------------------------- 1 file changed, 11 insertions(+), 67 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 8758dcb..57587a5 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -21,20 +21,17 @@ import copy import logging import StringIO import cStringIO -import os import time import re from collections import defaultdict, namedtuple from email.parser import Parser -from functools import wraps from zope.interface import implements from zope.proxy import sameProxiedObjects from twisted.mail import imap4 from twisted.internet import defer -from twisted.internet.threads import deferToThread from twisted.python import log from u1db import errors as u1db_errors @@ -44,70 +41,12 @@ from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL from leap.common.check import leap_assert, leap_assert_type from leap.common.mail import get_email_charset from leap.mail.messageflow import IMessageConsumer, MessageProducer +from leap.mail.decorators import deferred from leap.soledad.client import Soledad logger = logging.getLogger(__name__) -def deferred(f): - ''' - Decorator, for deferring methods to Threads. - - It will do a deferToThread of the decorated method - unless the environment variable LEAPMAIL_DEBUG is set. - - It uses a descriptor to delay the definition of the - method wrapper. - ''' - class descript(object): - def __init__(self, f): - self.f = f - - def __get__(self, instance, klass): - if instance is None: - # Class method was requested - return self.make_unbound(klass) - return self.make_bound(instance) - - def _errback(self, failure): - err = failure.value - logger.warning('error in method: %s' % (self.f.__name__)) - log.err(err) - - def make_unbound(self, klass): - - @wraps(self.f) - def wrapper(*args, **kwargs): - '''This documentation will vanish :)''' - raise TypeError( - 'unbound method {}() must be called with {} instance ' - 'as first argument (got nothing instead)'.format( - self.f.__name__, - klass.__name__) - ) - return wrapper - - def make_bound(self, instance): - - @wraps(self.f) - def wrapper(*args, **kwargs): - '''This documentation will disapear :)''' - - if not os.environ.get('LEAPMAIL_DEBUG'): - d = deferToThread(self.f, instance, *args, **kwargs) - d.addErrback(self._errback) - return d - else: - return self.f(instance, *args, **kwargs) - - # This instance does not need the descriptor anymore, - # let it find the wrapper directly next time: - setattr(instance, self.f.__name__, wrapper) - return wrapper - - return descript(f) - - class MissingIndexError(Exception): """ Raises when tried to access a non existent index document. @@ -248,6 +187,8 @@ class MailParser(object): return self._parser.parse if isinstance(o, basestring): return self._parser.parsestr + # fallback + return self._parser.parsestr def _stringify(self, o): """ @@ -942,8 +883,8 @@ class LeapMessage(fields, MailParser, MBoxParser): Return True if this message is multipart. """ if self._cdoc: - retval = self._cdoc.content.get(self.MULTIPART_KEY, False) - print "MULTIPART? ", retval + retval = self._fdoc.content.get(self.MULTIPART_KEY, False) + return retval def getSubPart(self, part): """ @@ -1197,6 +1138,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): msg = self._get_parsed_msg(raw) headers = dict(msg) + logger.debug("adding. is multipart:%s" % msg.is_multipart()) flags_doc[self.MULTIPART_KEY] = msg.is_multipart() # XXX get lower case for keys? # XXX get headers doc @@ -1464,7 +1406,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def addListener(self, listener): """ - Rdds a listener to the listeners queue. + Adds a listener to the listeners queue. + The server adds itself as a listener when there is a SELECT, + so it can send EXIST commands. :param listener: listener to add :type listener: an object that implements IMailboxListener @@ -1716,6 +1660,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: a deferred that evals to None """ # XXX we should treat the message as an IMessage from here + leap_assert_type(message, basestring) uid_next = self.getUIDNext() logger.debug('Adding msg with UID :%s' % uid_next) if flags is None: @@ -1823,12 +1768,11 @@ class SoledadMailbox(WithMsgFields, MBoxParser): else: for msg_id in messages: - print "getting msg by uid", msg_id msg = self.messages.get_msg_by_uid(msg_id) if msg: result.append((msg_id, msg)) else: - print "fetch %s, no msg found!!!" % msg_id + logger.debug("fetch %s, no msg found!!!" % msg_id) if self.isWriteable(): self._unset_recent_flag() -- cgit v1.2.3 From 62b0cd6301b7097dfa2776b677ab3c7d27f60d7b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Dec 2013 14:10:14 -0400 Subject: Split the near-2k loc file into more handy modules. ...aaaand not a single fuck was given that day! --- src/leap/mail/imap/server.py | 1897 ------------------------------------------ 1 file changed, 1897 deletions(-) delete mode 100644 src/leap/mail/imap/server.py (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py deleted file mode 100644 index 57587a5..0000000 --- a/src/leap/mail/imap/server.py +++ /dev/null @@ -1,1897 +0,0 @@ -# -*- coding: utf-8 -*- -# server.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Soledad-backed IMAP Server. -""" -import copy -import logging -import StringIO -import cStringIO -import time -import re - -from collections import defaultdict, namedtuple -from email.parser import Parser - -from zope.interface import implements -from zope.proxy import sameProxiedObjects - -from twisted.mail import imap4 -from twisted.internet import defer -from twisted.python import log - -from u1db import errors as u1db_errors - -from leap.common import events as leap_events -from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL -from leap.common.check import leap_assert, leap_assert_type -from leap.common.mail import get_email_charset -from leap.mail.messageflow import IMessageConsumer, MessageProducer -from leap.mail.decorators import deferred -from leap.soledad.client import Soledad - -logger = logging.getLogger(__name__) - - -class MissingIndexError(Exception): - """ - Raises when tried to access a non existent index document. - """ - - -class BadIndexError(Exception): - """ - Raises when index is malformed or has the wrong cardinality. - """ - - -class WithMsgFields(object): - """ - Container class for class-attributes to be shared by - several message-related classes. - """ - # Internal representation of Message - DATE_KEY = "date" - HEADERS_KEY = "headers" - FLAGS_KEY = "flags" - MBOX_KEY = "mbox" - RAW_KEY = "raw" - SUBJECT_KEY = "subject" - UID_KEY = "uid" - MULTIPART_KEY = "multi" - SIZE_KEY = "size" - - # Mailbox specific keys - CLOSED_KEY = "closed" - CREATED_KEY = "created" - SUBSCRIBED_KEY = "subscribed" - RW_KEY = "rw" - LAST_UID_KEY = "lastuid" - - # Document Type, for indexing - TYPE_KEY = "type" - TYPE_MESSAGE_VAL = "msg" - TYPE_MBOX_VAL = "mbox" - TYPE_FLAGS_VAL = "flags" - # should add also a headers val - - INBOX_VAL = "inbox" - - # Flags for SoledadDocument for indexing. - SEEN_KEY = "seen" - RECENT_KEY = "recent" - - # Flags in Mailbox and Message - SEEN_FLAG = "\\Seen" - RECENT_FLAG = "\\Recent" - ANSWERED_FLAG = "\\Answered" - FLAGGED_FLAG = "\\Flagged" # yo dawg - DELETED_FLAG = "\\Deleted" - DRAFT_FLAG = "\\Draft" - NOSELECT_FLAG = "\\Noselect" - LIST_FLAG = "List" # is this OK? (no \. ie, no system flag) - - # Fields in mail object - SUBJECT_FIELD = "Subject" - DATE_FIELD = "Date" - -fields = WithMsgFields # alias for convenience - - -class IndexedDB(object): - """ - Methods dealing with the index. - - This is a MixIn that needs access to the soledad instance, - and also assumes that a INDEXES attribute is accessible to the instance. - - INDEXES must be a dictionary of type: - {'index-name': ['field1', 'field2']} - """ - # TODO we might want to move this to soledad itself, check - - def initialize_db(self): - """ - Initialize the database. - """ - leap_assert(self._soledad, - "Need a soledad attribute accesible in the instance") - leap_assert_type(self.INDEXES, dict) - - # Ask the database for currently existing indexes. - if not self._soledad: - logger.debug("NO SOLEDAD ON IMAP INITIALIZATION") - return - db_indexes = dict() - if self._soledad is not None: - db_indexes = dict(self._soledad.list_indexes()) - for name, expression in SoledadBackedAccount.INDEXES.items(): - if name not in db_indexes: - # The index does not yet exist. - self._soledad.create_index(name, *expression) - continue - - if expression == db_indexes[name]: - # The index exists and is up to date. - continue - # The index exists but the definition is not what expected, so we - # delete it and add the proper index expression. - self._soledad.delete_index(name) - self._soledad.create_index(name, *expression) - - -class MailParser(object): - """ - Mixin with utility methods to parse raw messages. - """ - def __init__(self): - """ - Initializes the mail parser. - """ - self._parser = Parser() - - def _get_parsed_msg(self, raw): - """ - Return a parsed Message. - - :param raw: the raw string to parse - :type raw: basestring, or StringIO object - """ - msg = self._get_parser_fun(raw)(raw, True) - return msg - - def _get_parser_fun(self, o): - """ - Retunn the proper parser function for an object. - - :param o: object - :type o: object - :param parser: an instance of email.parser.Parser - :type parser: email.parser.Parser - """ - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return self._parser.parse - if isinstance(o, basestring): - return self._parser.parsestr - # fallback - return self._parser.parsestr - - def _stringify(self, o): - """ - Return a string object. - - :param o: object - :type o: object - """ - if isinstance(o, (cStringIO.OutputType, StringIO.StringIO)): - return o.getvalue() - else: - return o - - -class MBoxParser(object): - """ - Utility function to parse mailbox names. - """ - INBOX_NAME = "INBOX" - INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE) - - def _parse_mailbox_name(self, name): - """ - :param name: the name of the mailbox - :type name: unicode - - :rtype: unicode - """ - if self.INBOX_RE.match(name): - # ensure inital INBOX is uppercase - return self.INBOX_NAME + name[len(self.INBOX_NAME):] - return name - - -####################################### -# Soledad Account -####################################### - - -class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser): - """ - An implementation of IAccount and INamespacePresenteer - that is backed by Soledad Encrypted Documents. - """ - - implements(imap4.IAccount, imap4.INamespacePresenter) - - _soledad = None - selected = None - - TYPE_IDX = 'by-type' - TYPE_MBOX_IDX = 'by-type-and-mbox' - TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid' - TYPE_SUBS_IDX = 'by-type-and-subscribed' - TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen' - TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent' - # Tomas created the `recent and seen index`, but the semantic is not too - # correct since the recent flag is volatile. - TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen' - - KTYPE = WithMsgFields.TYPE_KEY - MBOX_VAL = WithMsgFields.TYPE_MBOX_VAL - - INDEXES = { - # generic - TYPE_IDX: [KTYPE], - TYPE_MBOX_IDX: [KTYPE, MBOX_VAL], - TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, WithMsgFields.UID_KEY], - - # mailboxes - TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'], - - # messages - TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'], - TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'], - TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL, - 'bool(recent)', 'bool(seen)'], - } - - MBOX_KEY = MBOX_VAL - - EMPTY_MBOX = { - WithMsgFields.TYPE_KEY: MBOX_KEY, - WithMsgFields.TYPE_MBOX_VAL: MBoxParser.INBOX_NAME, - WithMsgFields.SUBJECT_KEY: "", - WithMsgFields.FLAGS_KEY: [], - WithMsgFields.CLOSED_KEY: False, - WithMsgFields.SUBSCRIBED_KEY: False, - WithMsgFields.RW_KEY: 1, - WithMsgFields.LAST_UID_KEY: 0 - } - - def __init__(self, account_name, soledad=None): - """ - Creates a SoledadAccountIndex that keeps track of the mailboxes - and subscriptions handled by this account. - - :param acct_name: The name of the account (user id). - :type acct_name: str - - :param soledad: a Soledad instance. - :param soledad: Soledad - """ - leap_assert(soledad, "Need a soledad instance to initialize") - leap_assert_type(soledad, Soledad) - - # XXX SHOULD assert too that the name matches the user/uuid with which - # soledad has been initialized. - - self._account_name = self._parse_mailbox_name(account_name) - self._soledad = soledad - - self.initialize_db() - - # every user should have the right to an inbox folder - # at least, so let's make one! - - if not self.mailboxes: - self.addMailbox(self.INBOX_NAME) - - def _get_empty_mailbox(self): - """ - Returns an empty mailbox. - - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MBOX) - - def _get_mailbox_by_name(self, name): - """ - Return an mbox document by name. - - :param name: the name of the mailbox - :type name: str - - :rtype: SoledadDocument - """ - doc = self._soledad.get_from_index( - self.TYPE_MBOX_IDX, self.MBOX_KEY, - self._parse_mailbox_name(name)) - return doc[0] if doc else None - - @property - def mailboxes(self): - """ - A list of the current mailboxes for this account. - """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_IDX, self.MBOX_KEY)] - - @property - def subscriptions(self): - """ - A list of the current subscriptions for this account. - """ - return [doc.content[self.MBOX_KEY] - for doc in self._soledad.get_from_index( - self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')] - - def getMailbox(self, name): - """ - Returns a Mailbox with that name, without selecting it. - - :param name: name of the mailbox - :type name: str - - :returns: a a SoledadMailbox instance - :rtype: SoledadMailbox - """ - name = self._parse_mailbox_name(name) - - if name not in self.mailboxes: - raise imap4.MailboxException("No such mailbox") - - return SoledadMailbox(name, soledad=self._soledad) - - ## - ## IAccount - ## - - def addMailbox(self, name, creation_ts=None): - """ - Add a mailbox to the account. - - :param name: the name of the mailbox - :type name: str - - :param creation_ts: an optional creation timestamp to be used as - mailbox id. A timestamp will be used if no - one is provided. - :type creation_ts: int - - :returns: True if successful - :rtype: bool - """ - name = self._parse_mailbox_name(name) - - if name in self.mailboxes: - raise imap4.MailboxCollision, name - - if not creation_ts: - # by default, we pass an int value - # taken from the current time - # we make sure to take enough decimals to get a unique - # mailbox-uidvalidity. - creation_ts = int(time.time() * 10E2) - - mbox = self._get_empty_mailbox() - mbox[self.MBOX_KEY] = name - mbox[self.CREATED_KEY] = creation_ts - - doc = self._soledad.create_doc(mbox) - return bool(doc) - - def create(self, pathspec): - """ - Create a new mailbox from the given hierarchical name. - - :param pathspec: The full hierarchical name of a new mailbox to create. - If any of the inferior hierarchical names to this one - do not exist, they are created as well. - :type pathspec: str - - :return: A true value if the creation succeeds. - :rtype: bool - - :raise MailboxException: Raised if this mailbox cannot be added. - """ - # TODO raise MailboxException - paths = filter( - None, - self._parse_mailbox_name(pathspec).split('/')) - for accum in range(1, len(paths)): - try: - self.addMailbox('/'.join(paths[:accum])) - except imap4.MailboxCollision: - pass - try: - self.addMailbox('/'.join(paths)) - except imap4.MailboxCollision: - if not pathspec.endswith('/'): - return False - return True - - def select(self, name, readwrite=1): - """ - Selects a mailbox. - - :param name: the mailbox to select - :type name: str - - :param readwrite: 1 for readwrite permissions. - :type readwrite: int - - :rtype: bool - """ - name = self._parse_mailbox_name(name) - - if name not in self.mailboxes: - return None - - self.selected = name - - return SoledadMailbox( - name, rw=readwrite, - soledad=self._soledad) - - def delete(self, name, force=False): - """ - Deletes a mailbox. - - Right now it does not purge the messages, but just removes the mailbox - name from the mailboxes list!!! - - :param name: the mailbox to be deleted - :type name: str - - :param force: if True, it will not check for noselect flag or inferior - names. use with care. - :type force: bool - """ - name = self._parse_mailbox_name(name) - - if not name in self.mailboxes: - raise imap4.MailboxException("No such mailbox") - - mbox = self.getMailbox(name) - - if force is False: - # See if this box is flagged \Noselect - # XXX use mbox.flags instead? - if self.NOSELECT_FLAG in mbox.getFlags(): - # Check for hierarchically inferior mailboxes with this one - # as part of their root. - for others in self.mailboxes: - if others != name and others.startswith(name): - raise imap4.MailboxException, ( - "Hierarchically inferior mailboxes " - "exist and \\Noselect is set") - mbox.destroy() - - # XXX FIXME --- not honoring the inferior names... - - # if there are no hierarchically inferior names, we will - # delete it from our ken. - #if self._inferiorNames(name) > 1: - # ??! -- can this be rite? - #self._index.removeMailbox(name) - - def rename(self, oldname, newname): - """ - Renames a mailbox. - - :param oldname: old name of the mailbox - :type oldname: str - - :param newname: new name of the mailbox - :type newname: str - """ - oldname = self._parse_mailbox_name(oldname) - newname = self._parse_mailbox_name(newname) - - if oldname not in self.mailboxes: - raise imap4.NoSuchMailbox, oldname - - inferiors = self._inferiorNames(oldname) - inferiors = [(o, o.replace(oldname, newname, 1)) for o in inferiors] - - for (old, new) in inferiors: - if new in self.mailboxes: - raise imap4.MailboxCollision, new - - for (old, new) in inferiors: - mbox = self._get_mailbox_by_name(old) - mbox.content[self.MBOX_KEY] = new - self._soledad.put_doc(mbox) - - # XXX ---- FIXME!!!! ------------------------------------ - # until here we just renamed the index... - # We have to rename also the occurrence of this - # mailbox on ALL the messages that are contained in it!!! - # ... we maybe could use a reference to the doc_id - # in each msg, instead of the "mbox" field in msgs - # ------------------------------------------------------- - - def _inferiorNames(self, name): - """ - Return hierarchically inferior mailboxes. - - :param name: name of the mailbox - :rtype: list - """ - # XXX use wildcard query instead - inferiors = [] - for infname in self.mailboxes: - if infname.startswith(name): - inferiors.append(infname) - return inferiors - - def isSubscribed(self, name): - """ - Returns True if user is subscribed to this mailbox. - - :param name: the mailbox to be checked. - :type name: str - - :rtype: bool - """ - mbox = self._get_mailbox_by_name(name) - return mbox.content.get('subscribed', False) - - def _set_subscription(self, name, value): - """ - Sets the subscription value for a given mailbox - - :param name: the mailbox - :type name: str - - :param value: the boolean value - :type value: bool - """ - # maybe we should store subscriptions in another - # document... - if not name in self.mailboxes: - self.addMailbox(name) - mbox = self._get_mailbox_by_name(name) - - if mbox: - mbox.content[self.SUBSCRIBED_KEY] = value - self._soledad.put_doc(mbox) - - def subscribe(self, name): - """ - Subscribe to this mailbox - - :param name: name of the mailbox - :type name: str - """ - name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - self._set_subscription(name, True) - - def unsubscribe(self, name): - """ - Unsubscribe from this mailbox - - :param name: name of the mailbox - :type name: str - """ - name = self._parse_mailbox_name(name) - if name not in self.subscriptions: - raise imap4.MailboxException, "Not currently subscribed to " + name - self._set_subscription(name, False) - - def listMailboxes(self, ref, wildcard): - """ - List the mailboxes. - - from rfc 3501: - returns a subset of names from the complete set - of all names available to the client. Zero or more untagged LIST - replies are returned, containing the name attributes, hierarchy - delimiter, and name. - - :param ref: reference name - :type ref: str - - :param wildcard: mailbox name with possible wildcards - :type wildcard: str - """ - # XXX use wildcard in index query - ref = self._inferiorNames( - self._parse_mailbox_name(ref)) - wildcard = imap4.wildcardToRegexp(wildcard, '/') - return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)] - - ## - ## INamespacePresenter - ## - - def getPersonalNamespaces(self): - return [["", "/"]] - - def getSharedNamespaces(self): - return None - - def getOtherNamespaces(self): - return None - - # extra, for convenience - - def deleteAllMessages(self, iknowhatiamdoing=False): - """ - Deletes all messages from all mailboxes. - Danger! high voltage! - - :param iknowhatiamdoing: confirmation parameter, needs to be True - to proceed. - """ - if iknowhatiamdoing is True: - for mbox in self.mailboxes: - self.delete(mbox, force=True) - - def __repr__(self): - """ - Representation string for this object. - """ - return "" % self._account_name - -####################################### -# LeapMessage, MessageCollection -# and Mailbox -####################################### - - -class LeapMessage(fields, MailParser, MBoxParser): - - implements(imap4.IMessage) - - def __init__(self, soledad, uid, mbox): - """ - Initializes a LeapMessage. - - :param soledad: a Soledad instance - :type soledad: Soledad - :param uid: the UID for the message. - :type uid: int or basestring - :param mbox: the mbox this message belongs to - :type mbox: basestring - """ - MailParser.__init__(self) - self._soledad = soledad - self._uid = int(uid) - self._mbox = self._parse_mailbox_name(mbox) - - self.__cdoc = None - - @property - def _fdoc(self): - """ - An accessor to the flags docuemnt - """ - return self._get_flags_doc() - - @property - def _cdoc(self): - """ - An accessor to the content docuemnt - """ - if not self.__cdoc: - self.__cdoc = self._get_content_doc() - return self.__cdoc - - def getUID(self): - """ - Retrieve the unique identifier associated with this message - - :return: uid for this message - :rtype: int - """ - return self._uid - - def getFlags(self): - """ - Retrieve the flags associated with this message - - :return: The flags, represented as strings - :rtype: tuple - """ - if self._uid is None: - return [] - - flags = [] - flag_doc = self._fdoc - if flag_doc: - flags = flag_doc.content.get(self.FLAGS_KEY, None) - if flags: - flags = map(str, flags) - return tuple(flags) - - # setFlags, addFlags, removeFlags are not in the interface spec - # but we use them with store command. - - def setFlags(self, flags): - """ - Sets the flags for this message - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to update in the message. - :type flags: tuple of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - log.msg('setting flags: %s' % (self._uid)) - - doc = self._fdoc - doc.content[self.FLAGS_KEY] = flags - doc.content[self.SEEN_KEY] = self.SEEN_FLAG in flags - doc.content[self.RECENT_KEY] = self.RECENT_FLAG in flags - self._soledad.put_doc(doc) - - def addFlags(self, flags): - """ - Adds flags to this message. - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to add to the message. - :type flags: tuple of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - oldflags = self.getFlags() - self.setFlags(tuple(set(flags + oldflags))) - - def removeFlags(self, flags): - """ - Remove flags from this message. - - Returns a SoledadDocument that needs to be updated by the caller. - - :param flags: the flags to be removed from the message. - :type flags: tuple of str - - :return: a SoledadDocument instance - :rtype: SoledadDocument - """ - leap_assert(isinstance(flags, tuple), "flags need to be a tuple") - oldflags = self.getFlags() - self.setFlags(tuple(set(oldflags) - set(flags))) - - def getInternalDate(self): - """ - Retrieve the date internally associated with this message - - :rtype: C{str} - :return: An RFC822-formatted date string. - """ - return str(self._cdoc.content.get(self.DATE_KEY, '')) - - # - # IMessagePart - # - - # XXX we should implement this interface too for the subparts - # so we allow nested parts... - - def getBodyFile(self): - """ - Retrieve a file object containing only the body of this message. - - :return: file-like object opened for reading - :rtype: StringIO - """ - fd = StringIO.StringIO() - - cdoc = self._cdoc - content = cdoc.content.get(self.RAW_KEY, '') - charset = get_email_charset( - unicode(cdoc.content.get(self.RAW_KEY, ''))) - try: - content = content.encode(charset) - except (UnicodeEncodeError, UnicodeDecodeError) as e: - logger.error("Unicode error {0}".format(e)) - content = content.encode(charset, 'replace') - - raw = self._get_raw_msg() - msg = self._get_parsed_msg(raw) - body = msg.get_payload() - fd.write(body) - # XXX SHOULD use a separate BODY FIELD ... - fd.seek(0) - return fd - - def getSize(self): - """ - Return the total size, in octets, of this message. - - :return: size of the message, in octets - :rtype: int - """ - size = self._cdoc.content.get(self.SIZE_KEY, False) - if not size: - # XXX fallback, should remove when all migrated. - size = self.getBodyFile().len - return size - - def _get_headers(self): - """ - Return the headers dict stored in this message document. - """ - # XXX get from the headers doc - return self._cdoc.content.get(self.HEADERS_KEY, {}) - - def getHeaders(self, negate, *names): - """ - Retrieve a group of message headers. - - :param names: The names of the headers to retrieve or omit. - :type names: tuple of str - - :param negate: If True, indicates that the headers listed in names - should be omitted from the return value, rather - than included. - :type negate: bool - - :return: A mapping of header field names to header field values - :rtype: dict - """ - headers = self._get_headers() - names = map(lambda s: s.upper(), names) - if negate: - cond = lambda key: key.upper() not in names - else: - cond = lambda key: key.upper() in names - - # unpack and filter original dict by negate-condition - filter_by_cond = [ - map(str, (key, val)) for - key, val in headers.items() - if cond(key)] - return dict(filter_by_cond) - - def isMultipart(self): - """ - Return True if this message is multipart. - """ - if self._cdoc: - retval = self._fdoc.content.get(self.MULTIPART_KEY, False) - return retval - - def getSubPart(self, part): - """ - Retrieve a MIME submessage - - :type part: C{int} - :param part: The number of the part to retrieve, indexed from 0. - :raise IndexError: Raised if the specified part does not exist. - :raise TypeError: Raised if this message is not multipart. - :rtype: Any object implementing C{IMessagePart}. - :return: The specified sub-part. - """ - if not self.isMultipart(): - raise TypeError - - msg = self._get_parsed_msg() - # XXX should wrap IMessagePart - return msg.get_payload()[part] - - # - # accessors - # - - def _get_flags_doc(self): - """ - Return the document that keeps the flags for this - message. - """ - flag_docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, - fields.TYPE_FLAGS_VAL, self._mbox, str(self._uid)) - flag_doc = flag_docs[0] if flag_docs else None - return flag_doc - - def _get_content_doc(self): - """ - Return the document that keeps the flags for this - message. - """ - cont_docs = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_UID_IDX, - fields.TYPE_MESSAGE_VAL, self._mbox, str(self._uid)) - cont_doc = cont_docs[0] if cont_docs else None - return cont_doc - - def _get_raw_msg(self): - """ - Return the raw msg. - :rtype: basestring - """ - return self._cdoc.content.get(self.RAW_KEY, '') - - def __getitem__(self, key): - """ - Return the content of the message document. - - :param key: The key - :type key: str - - :return: The content value indexed by C{key} or None - :rtype: str - """ - return self._cdoc.content.get(key, None) - - def does_exist(self): - """ - Return True if there is actually a message for this - UID and mbox. - """ - return bool(self._fdoc) - - -SoledadWriterPayload = namedtuple( - 'SoledadWriterPayload', ['mode', 'payload']) - -SoledadWriterPayload.CREATE = 1 -SoledadWriterPayload.PUT = 2 - - -class SoledadDocWriter(object): - """ - This writer will create docs serially in the local soledad database. - """ - - implements(IMessageConsumer) - - def __init__(self, soledad): - """ - Initialize the writer. - - :param soledad: the soledad instance - :type soledad: Soledad - """ - self._soledad = soledad - - def consume(self, queue): - """ - Creates a new document in soledad db. - - :param queue: queue to get item from, with content of the document - to be inserted. - :type queue: Queue - """ - empty = queue.empty() - while not empty: - item = queue.get() - if item.mode == SoledadWriterPayload.CREATE: - call = self._soledad.create_doc - elif item.mode == SoledadWriterPayload.PUT: - call = self._soledad.put_doc - - # should handle errors - try: - call(item.payload) - except u1db_errors.RevisionConflict as exc: - logger.error("Error: %r" % (exc,)) - raise exc - - empty = queue.empty() - - -class MessageCollection(WithMsgFields, IndexedDB, MailParser, MBoxParser): - """ - A collection of messages, surprisingly. - - It is tied to a selected mailbox name that is passed to constructor. - Implements a filter query over the messages contained in a soledad - database. - """ - # XXX this should be able to produce a MessageSet methinks - - EMPTY_MSG = { - fields.TYPE_KEY: fields.TYPE_MESSAGE_VAL, - fields.UID_KEY: 1, - fields.MBOX_KEY: fields.INBOX_VAL, - - fields.SUBJECT_KEY: "", - fields.DATE_KEY: "", - fields.RAW_KEY: "", - - # XXX should separate headers into another doc - fields.HEADERS_KEY: {}, - } - - EMPTY_FLAGS = { - fields.TYPE_KEY: fields.TYPE_FLAGS_VAL, - fields.UID_KEY: 1, - fields.MBOX_KEY: fields.INBOX_VAL, - - fields.FLAGS_KEY: [], - fields.SEEN_KEY: False, - fields.RECENT_KEY: True, - fields.MULTIPART_KEY: False, - } - - # get from SoledadBackedAccount the needed index-related constants - INDEXES = SoledadBackedAccount.INDEXES - TYPE_IDX = SoledadBackedAccount.TYPE_IDX - - def __init__(self, mbox=None, soledad=None): - """ - Constructor for MessageCollection. - - :param mbox: the name of the mailbox. It is the name - with which we filter the query over the - messages database - :type mbox: str - - :param soledad: Soledad database - :type soledad: Soledad instance - """ - MailParser.__init__(self) - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(mbox.strip() != "", "mbox cannot be blank space") - leap_assert(isinstance(mbox, (str, unicode)), - "mbox needs to be a string") - leap_assert(soledad, "Need a soledad instance to initialize") - - # okay, all in order, keep going... - self.mbox = self._parse_mailbox_name(mbox) - self._soledad = soledad - self.initialize_db() - - # I think of someone like nietzsche when reading this - - # this will be the producer that will enqueue the content - # to be processed serially by the consumer (the writer). We just - # need to `put` the new material on its plate. - - self.soledad_writer = MessageProducer( - SoledadDocWriter(soledad), - period=0.05) - - def _get_empty_msg(self): - """ - Returns an empty message. - - :return: a dict containing a default empty message - :rtype: dict - """ - return copy.deepcopy(self.EMPTY_MSG) - - def _get_empty_flags_doc(self): - """ - Returns an empty doc for storing flags. - - :return: - :rtype: - """ - return copy.deepcopy(self.EMPTY_FLAGS) - - @deferred - def add_msg(self, raw, subject=None, flags=None, date=None, uid=1): - """ - Creates a new message document. - - :param raw: the raw message - :type raw: str - - :param subject: subject of the message. - :type subject: str - - :param flags: flags - :type flags: list - - :param date: the received date for the message - :type date: str - - :param uid: the message uid for this mailbox - :type uid: int - """ - # TODO: split in smaller methods - logger.debug('adding message') - if flags is None: - flags = tuple() - leap_assert_type(flags, tuple) - - content_doc = self._get_empty_msg() - flags_doc = self._get_empty_flags_doc() - - content_doc[self.MBOX_KEY] = self.mbox - flags_doc[self.MBOX_KEY] = self.mbox - # ...should get a sanity check here. - content_doc[self.UID_KEY] = uid - flags_doc[self.UID_KEY] = uid - - if flags: - flags_doc[self.FLAGS_KEY] = map(self._stringify, flags) - flags_doc[self.SEEN_KEY] = self.SEEN_FLAG in flags - - msg = self._get_parsed_msg(raw) - headers = dict(msg) - - logger.debug("adding. is multipart:%s" % msg.is_multipart()) - flags_doc[self.MULTIPART_KEY] = msg.is_multipart() - # XXX get lower case for keys? - # XXX get headers doc - content_doc[self.HEADERS_KEY] = headers - # set subject based on message headers and eventually replace by - # subject given as param - if self.SUBJECT_FIELD in headers: - content_doc[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD] - if subject is not None: - content_doc[self.SUBJECT_KEY] = subject - - # XXX could separate body into its own doc - # but should also separate multiparts - # that should be wrapped in MessagePart - content_doc[self.RAW_KEY] = self._stringify(raw) - content_doc[self.SIZE_KEY] = len(raw) - - if not date and self.DATE_FIELD in headers: - content_doc[self.DATE_KEY] = headers[self.DATE_FIELD] - else: - content_doc[self.DATE_KEY] = date - - logger.debug('enqueuing message for write') - - ptuple = SoledadWriterPayload - self.soledad_writer.put(ptuple( - mode=ptuple.CREATE, payload=content_doc)) - self.soledad_writer.put(ptuple( - mode=ptuple.CREATE, payload=flags_doc)) - - def remove(self, msg): - """ - Removes a message. - - :param msg: a u1db doc containing the message - :type msg: SoledadDocument - """ - self._soledad.delete_doc(msg) - - # getters - - def get_msg_by_uid(self, uid): - """ - Retrieves a LeapMessage by UID. - - :param uid: the message uid to query by - :type uid: int - - :return: A LeapMessage instance matching the query, - or None if not found. - :rtype: LeapMessage - """ - msg = LeapMessage(self._soledad, uid, self.mbox) - if not msg.does_exist(): - return None - return msg - - def get_all(self): - """ - Get all message documents for the selected mailbox. - If you want acess to the content, use __iter__ instead - - :return: a list of u1db documents - :rtype: list of SoledadDocument - """ - # TODO change to get_all_docs and turn this - # into returning messages - if sameProxiedObjects(self._soledad, None): - logger.warning('Tried to get messages but soledad is None!') - return [] - - all_docs = [doc for doc in self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox)] - - # inneficient, but first let's grok it and then - # let's worry about efficiency. - # XXX FIXINDEX -- should implement order by in soledad - return sorted(all_docs, key=lambda item: item.content['uid']) - - def count(self): - """ - Return the count of messages for this mailbox. - - :rtype: int - """ - count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - fields.TYPE_FLAGS_VAL, self.mbox) - return count - - # unseen messages - - def unseen_iter(self): - """ - Get an iterator for the message docs with no `seen` flag - - :return: iterator through unseen message doc UIDs - :rtype: iterable - """ - return (doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '0')) - - def count_unseen(self): - """ - Count all messages with the `Unseen` flag. - - :returns: count - :rtype: int - """ - count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_SEEN_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '0') - return count - - def get_unseen(self): - """ - Get all messages with the `Unseen` flag - - :returns: a list of LeapMessages - :rtype: list - """ - return [LeapMessage(self._soledad, docid, self.mbox) - for docid in self.unseen_iter()] - - # recent messages - - def recent_iter(self): - """ - Get an iterator for the message docs with `recent` flag. - - :return: iterator through recent message docs - :rtype: iterable - """ - return (doc.content[self.UID_KEY] for doc in - self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '1')) - - def get_recent(self): - """ - Get all messages with the `Recent` flag. - - :returns: a list of LeapMessages - :rtype: list - """ - return [LeapMessage(self._soledad, docid, self.mbox) - for docid in self.recent_iter()] - - def count_recent(self): - """ - Count all messages with the `Recent` flag. - - :returns: count - :rtype: int - """ - count = self._soledad.get_count_from_index( - SoledadBackedAccount.TYPE_MBOX_RECT_IDX, - self.TYPE_FLAGS_VAL, self.mbox, '1') - return count - - def __len__(self): - """ - Returns the number of messages on this mailbox. - - :rtype: int - """ - return self.count() - - def __iter__(self): - """ - Returns an iterator over all messages. - - :returns: iterator of dicts with content for all messages. - :rtype: iterable - """ - # XXX return LeapMessage instead?! (change accordingly) - return (m.content for m in self.get_all()) - - def __repr__(self): - """ - Representation string for this object. - """ - return u"" % ( - self.mbox, self.count()) - - # XXX should implement __eq__ also !!! --- use a hash - # of content for that, will be used for dedup. - - -class SoledadMailbox(WithMsgFields, MBoxParser): - """ - A Soledad-backed IMAP mailbox. - - Implements the high-level method needed for the Mailbox interfaces. - The low-level database methods are contained in MessageCollection class, - which we instantiate and make accessible in the `messages` attribute. - """ - implements(imap4.IMailboxInfo, imap4.IMailbox, imap4.ICloseableMailbox) - # XXX should finish the implementation of IMailboxListener - - messages = None - _closed = False - - INIT_FLAGS = (WithMsgFields.SEEN_FLAG, WithMsgFields.ANSWERED_FLAG, - WithMsgFields.FLAGGED_FLAG, WithMsgFields.DELETED_FLAG, - WithMsgFields.DRAFT_FLAG, WithMsgFields.RECENT_FLAG, - WithMsgFields.LIST_FLAG) - flags = None - - CMD_MSG = "MESSAGES" - CMD_RECENT = "RECENT" - CMD_UIDNEXT = "UIDNEXT" - CMD_UIDVALIDITY = "UIDVALIDITY" - CMD_UNSEEN = "UNSEEN" - - _listeners = defaultdict(set) - - def __init__(self, mbox, soledad=None, rw=1): - """ - SoledadMailbox constructor. Needs to get passed a name, plus a - Soledad instance. - - :param mbox: the mailbox name - :type mbox: str - - :param soledad: a Soledad instance. - :type soledad: Soledad - - :param rw: read-and-write flags - :type rw: int - """ - leap_assert(mbox, "Need a mailbox name to initialize") - leap_assert(soledad, "Need a soledad instance to initialize") - - # XXX should move to wrapper - #leap_assert(isinstance(soledad._db, SQLCipherDatabase), - #"soledad._db must be an instance of SQLCipherDatabase") - - self.mbox = self._parse_mailbox_name(mbox) - self.rw = rw - - self._soledad = soledad - - self.messages = MessageCollection( - mbox=mbox, soledad=self._soledad) - - if not self.getFlags(): - self.setFlags(self.INIT_FLAGS) - - @property - def listeners(self): - """ - Returns listeners for this mbox. - - The server itself is a listener to the mailbox. - so we can notify it (and should!) after changes in flags - and number of messages. - - :rtype: set - """ - return self._listeners[self.mbox] - - def addListener(self, listener): - """ - Adds a listener to the listeners queue. - The server adds itself as a listener when there is a SELECT, - so it can send EXIST commands. - - :param listener: listener to add - :type listener: an object that implements IMailboxListener - """ - logger.debug('adding mailbox listener: %s' % listener) - self.listeners.add(listener) - - def removeListener(self, listener): - """ - Removes a listener from the listeners queue. - - :param listener: listener to remove - :type listener: an object that implements IMailboxListener - """ - self.listeners.remove(listener) - - def _get_mbox(self): - """ - Returns mailbox document. - - :return: A SoledadDocument containing this mailbox, or None if - the query failed. - :rtype: SoledadDocument or None. - """ - try: - query = self._soledad.get_from_index( - SoledadBackedAccount.TYPE_MBOX_IDX, - self.TYPE_MBOX_VAL, self.mbox) - if query: - return query.pop() - except Exception as exc: - logger.error("Unhandled error %r" % exc) - - def getFlags(self): - """ - Returns the flags defined for this mailbox. - - :returns: tuple of flags for this mailbox - :rtype: tuple of str - """ - mbox = self._get_mbox() - if not mbox: - return None - flags = mbox.content.get(self.FLAGS_KEY, []) - return map(str, flags) - - def setFlags(self, flags): - """ - Sets flags for this mailbox. - - :param flags: a tuple with the flags - :type flags: tuple of str - """ - leap_assert(isinstance(flags, tuple), - "flags expected to be a tuple") - mbox = self._get_mbox() - if not mbox: - return None - mbox.content[self.FLAGS_KEY] = map(str, flags) - self._soledad.put_doc(mbox) - - # XXX SHOULD BETTER IMPLEMENT ADD_FLAG, REMOVE_FLAG. - - def _get_closed(self): - """ - Return the closed attribute for this mailbox. - - :return: True if the mailbox is closed - :rtype: bool - """ - mbox = self._get_mbox() - return mbox.content.get(self.CLOSED_KEY, False) - - def _set_closed(self, closed): - """ - Set the closed attribute for this mailbox. - - :param closed: the state to be set - :type closed: bool - """ - leap_assert(isinstance(closed, bool), "closed needs to be boolean") - mbox = self._get_mbox() - mbox.content[self.CLOSED_KEY] = closed - self._soledad.put_doc(mbox) - - closed = property( - _get_closed, _set_closed, doc="Closed attribute.") - - def _get_last_uid(self): - """ - Return the last uid for this mailbox. - - :return: the last uid for messages in this mailbox - :rtype: bool - """ - mbox = self._get_mbox() - return mbox.content.get(self.LAST_UID_KEY, 1) - - def _set_last_uid(self, uid): - """ - Sets the last uid for this mailbox. - - :param uid: the uid to be set - :type uid: int - """ - leap_assert(isinstance(uid, int), "uid has to be int") - mbox = self._get_mbox() - key = self.LAST_UID_KEY - - count = self.getMessageCount() - - # XXX safety-catch. If we do get duplicates, - # we want to avoid further duplication. - - if uid >= count: - value = uid - else: - # something is wrong, - # just set the last uid - # beyond the max msg count. - logger.debug("WRONG uid < count. Setting last uid to %s", count) - value = count - - mbox.content[key] = value - self._soledad.put_doc(mbox) - - last_uid = property( - _get_last_uid, _set_last_uid, doc="Last_UID attribute.") - - def getUIDValidity(self): - """ - Return the unique validity identifier for this mailbox. - - :return: unique validity identifier - :rtype: int - """ - mbox = self._get_mbox() - return mbox.content.get(self.CREATED_KEY, 1) - - def getUID(self, message): - """ - Return the UID of a message in the mailbox - - .. note:: this implementation does not make much sense RIGHT NOW, - but in the future will be useful to get absolute UIDs from - message sequence numbers. - - :param message: the message uid - :type message: int - - :rtype: int - """ - msg = self.messages.get_msg_by_uid(message) - return msg.getUID() - - def getUIDNext(self): - """ - Return the likely UID for the next message added to this - mailbox. Currently it returns the higher UID incremented by - one. - - We increment the next uid *each* time this function gets called. - In this way, there will be gaps if the message with the allocated - uid cannot be saved. But that is preferable to having race conditions - if we get to parallel message adding. - - :rtype: int - """ - self.last_uid += 1 - return self.last_uid - - def getMessageCount(self): - """ - Returns the total count of messages in this mailbox. - - :rtype: int - """ - return self.messages.count() - - def getUnseenCount(self): - """ - Returns the number of messages with the 'Unseen' flag. - - :return: count of messages flagged `unseen` - :rtype: int - """ - return self.messages.count_unseen() - - def getRecentCount(self): - """ - Returns the number of messages with the 'Recent' flag. - - :return: count of messages flagged `recent` - :rtype: int - """ - return self.messages.count_recent() - - def isWriteable(self): - """ - Get the read/write status of the mailbox. - - :return: 1 if mailbox is read-writeable, 0 otherwise. - :rtype: int - """ - return self.rw - - def getHierarchicalDelimiter(self): - """ - Returns the character used to delimite hierarchies in mailboxes. - - :rtype: str - """ - return '/' - - def requestStatus(self, names): - """ - Handles a status request by gathering the output of the different - status commands. - - :param names: a list of strings containing the status commands - :type names: iter - """ - r = {} - if self.CMD_MSG in names: - r[self.CMD_MSG] = self.getMessageCount() - if self.CMD_RECENT in names: - r[self.CMD_RECENT] = self.getRecentCount() - if self.CMD_UIDNEXT in names: - r[self.CMD_UIDNEXT] = self.last_uid + 1 - if self.CMD_UIDVALIDITY in names: - r[self.CMD_UIDVALIDITY] = self.getUID() - if self.CMD_UNSEEN in names: - r[self.CMD_UNSEEN] = self.getUnseenCount() - return defer.succeed(r) - - def addMessage(self, message, flags, date=None): - """ - Adds a message to this mailbox. - - :param message: the raw message - :type message: str - - :param flags: flag list - :type flags: list of str - - :param date: timestamp - :type date: str - - :return: a deferred that evals to None - """ - # XXX we should treat the message as an IMessage from here - leap_assert_type(message, basestring) - uid_next = self.getUIDNext() - logger.debug('Adding msg with UID :%s' % uid_next) - if flags is None: - flags = tuple() - else: - flags = tuple(str(flag) for flag in flags) - - d = self._do_add_messages(message, flags, date, uid_next) - d.addCallback(self._notify_new) - - @deferred - def _do_add_messages(self, message, flags, date, uid_next): - """ - Calls to the messageCollection add_msg method (deferred to thread). - Invoked from addMessage. - """ - self.messages.add_msg(message, flags=flags, date=date, - uid=uid_next) - - def _notify_new(self, *args): - """ - Notify of new messages to all the listeners. - - :param args: ignored. - """ - exists = self.getMessageCount() - recent = self.getRecentCount() - logger.debug("NOTIFY: there are %s messages, %s recent" % ( - exists, - recent)) - - logger.debug("listeners: %s", str(self.listeners)) - for l in self.listeners: - logger.debug('notifying...') - l.newMessages(exists, recent) - - # commands, do not rename methods - - def destroy(self): - """ - Called before this mailbox is permanently deleted. - - Should cleanup resources, and set the \\Noselect flag - on the mailbox. - """ - self.setFlags((self.NOSELECT_FLAG,)) - self.deleteAllDocs() - - # XXX removing the mailbox in situ for now, - # we should postpone the removal - self._soledad.delete_doc(self._get_mbox()) - - def expunge(self): - """ - Remove all messages flagged \\Deleted - """ - if not self.isWriteable(): - raise imap4.ReadOnlyMailbox - delete = [] - deleted = [] - - for m in self.messages.get_all(): - if self.DELETED_FLAG in m.content[self.FLAGS_KEY]: - delete.append(m) - for m in delete: - deleted.append(m.content) - self.messages.remove(m) - - # XXX should return the UIDs of the deleted messages - # more generically - return [x for x in range(len(deleted))] - - @deferred - def fetch(self, messages, uid): - """ - Retrieve one or more messages in this mailbox. - - from rfc 3501: The data items to be fetched can be either a single atom - or a parenthesized list. - - :param messages: IDs of the messages to retrieve information about - :type messages: MessageSet - - :param uid: If true, the IDs are UIDs. They are message sequence IDs - otherwise. - :type uid: bool - - :rtype: A tuple of two-tuples of message sequence numbers and - LeapMessage - """ - result = [] - sequence = True if uid == 0 else False - - if not messages.last: - try: - iter(messages) - except TypeError: - # looks like we cannot iterate - messages.last = self.last_uid - - # for sequence numbers (uid = 0) - if sequence: - logger.debug("Getting msg by index: INEFFICIENT call!") - raise NotImplementedError - - else: - for msg_id in messages: - msg = self.messages.get_msg_by_uid(msg_id) - if msg: - result.append((msg_id, msg)) - else: - logger.debug("fetch %s, no msg found!!!" % msg_id) - - if self.isWriteable(): - self._unset_recent_flag() - self._signal_unread_to_ui() - - # XXX workaround for hangs in thunderbird - #return tuple(result[:100]) # --- doesn't show all!! - return tuple(result) - - @deferred - def _unset_recent_flag(self): - """ - Unsets `Recent` flag from a tuple of messages. - Called from fetch. - - From RFC, about `Recent`: - - Message is "recently" arrived in this mailbox. This session - is the first session to have been notified about this - message; if the session is read-write, subsequent sessions - will not see \Recent set for this message. This flag can not - be altered by the client. - - If it is not possible to determine whether or not this - session is the first session to be notified about a message, - then that message SHOULD be considered recent. - """ - log.msg('unsetting recent flags...') - for msg in self.messages.get_recent(): - msg.removeFlags((fields.RECENT_FLAG,)) - self._signal_unread_to_ui() - - @deferred - def _signal_unread_to_ui(self): - """ - Sends unread event to ui. - """ - unseen = self.getUnseenCount() - leap_events.signal(IMAP_UNREAD_MAIL, str(unseen)) - - @deferred - def store(self, messages, flags, mode, uid): - """ - Sets the flags of one or more messages. - - :param messages: The identifiers of the messages to set the flags - :type messages: A MessageSet object with the list of messages requested - - :param flags: The flags to set, unset, or add. - :type flags: sequence of str - - :param mode: If mode is -1, these flags should be removed from the - specified messages. If mode is 1, these flags should be - added to the specified messages. If mode is 0, all - existing flags should be cleared and these flags should be - added. - :type mode: -1, 0, or 1 - - :param uid: If true, the IDs specified in the query are UIDs; - otherwise they are message sequence IDs. - :type uid: bool - - :return: A dict mapping message sequence numbers to sequences of - str representing the flags set on the message after this - operation has been performed. - :rtype: dict - - :raise ReadOnlyMailbox: Raised if this mailbox is not open for - read-write. - """ - # XXX implement also sequence (uid = 0) - # XXX we should prevent cclient from setting Recent flag. - leap_assert(not isinstance(flags, basestring), - "flags cannot be a string") - flags = tuple(flags) - - if not self.isWriteable(): - log.msg('read only mailbox!') - raise imap4.ReadOnlyMailbox - - if not messages.last: - messages.last = self.messages.count() - - result = {} - for msg_id in messages: - log.msg("MSG ID = %s" % msg_id) - msg = self.messages.get_msg_by_uid(msg_id) - if mode == 1: - msg.addFlags(flags) - elif mode == -1: - msg.removeFlags(flags) - elif mode == 0: - msg.setFlags(flags) - result[msg_id] = msg.getFlags() - - self._signal_unread_to_ui() - return result - - @deferred - def close(self): - """ - Expunge and mark as closed - """ - self.expunge() - self.closed = True - - # convenience fun - - def deleteAllDocs(self): - """ - Deletes all docs in this mailbox - """ - docs = self.messages.get_all() - for doc in docs: - self.messages._soledad.delete_doc(doc) - - def __repr__(self): - """ - Representation string for this mailbox. - """ - return u"" % ( - self.mbox, self.messages.count()) -- cgit v1.2.3 From 23b2a2c10ddd6dbb15e3f532d526ac0a53bd788b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 23 Jan 2014 00:27:19 -0400 Subject: move server to its own file --- src/leap/mail/imap/server.py | 199 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 src/leap/mail/imap/server.py (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py new file mode 100644 index 0000000..8bd875b --- /dev/null +++ b/src/leap/mail/imap/server.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +# server.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Leap IMAP4 Server Implementation. +""" +from copy import copy + +from twisted import cred +from twisted.internet.defer import maybeDeferred +from twisted.internet.task import deferLater +from twisted.mail import imap4 +from twisted.python import log + +from leap.common import events as leap_events +from leap.common.check import leap_assert, leap_assert_type +from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN +from leap.soledad.client import Soledad + + +class LeapIMAPServer(imap4.IMAP4Server): + """ + An IMAP4 Server with mailboxes backed by soledad + """ + def __init__(self, *args, **kwargs): + # pop extraneous arguments + soledad = kwargs.pop('soledad', None) + uuid = kwargs.pop('uuid', None) + userid = kwargs.pop('userid', None) + leap_assert(soledad, "need a soledad instance") + leap_assert_type(soledad, Soledad) + leap_assert(uuid, "need a user in the initialization") + + self._userid = userid + + # initialize imap server! + imap4.IMAP4Server.__init__(self, *args, **kwargs) + + # we should initialize the account here, + # but we move it to the factory so we can + # populate the test account properly (and only once + # per session) + + def lineReceived(self, line): + """ + Attempt to parse a single line from the server. + + :param line: the line from the server, without the line delimiter. + :type line: str + """ + if self.theAccount.closed is True and self.state != "unauth": + log.msg("Closing the session. State: unauth") + self.state = "unauth" + + if "login" in line.lower(): + # avoid to log the pass, even though we are using a dummy auth + # by now. + msg = line[:7] + " [...]" + else: + msg = copy(line) + log.msg('rcv (%s): %s' % (self.state, msg)) + imap4.IMAP4Server.lineReceived(self, line) + + def authenticateLogin(self, username, password): + """ + Lookup the account with the given parameters, and deny + the improper combinations. + + :param username: the username that is attempting authentication. + :type username: str + :param password: the password to authenticate with. + :type password: str + """ + # XXX this should use portal: + # return portal.login(cred.credentials.UsernamePassword(user, pass) + if username != self._userid: + # bad username, reject. + raise cred.error.UnauthorizedLogin() + # any dummy password is allowed so far. use realm instead! + leap_events.signal(IMAP_CLIENT_LOGIN, "1") + return imap4.IAccount, self.theAccount, lambda: None + + def do_FETCH(self, tag, messages, query, uid=0): + """ + Overwritten fetch dispatcher to use the fast fetch_flags + method + """ + from twisted.internet import reactor + if not query: + self.sendPositiveResponse(tag, 'FETCH complete') + return # XXX ??? + + cbFetch = self._IMAP4Server__cbFetch + ebFetch = self._IMAP4Server__ebFetch + + if len(query) == 1 and str(query[0]) == "flags": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_flags, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + elif len(query) == 1 and str(query[0]) == "rfc822.header": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_headers, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + else: + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback( + ebFetch, tag) + + deferLater(reactor, + 2, self.mbox.unset_recent_flags, messages) + deferLater(reactor, 1, self.mbox.signal_unread_to_ui) + + select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, + imap4.IMAP4Server.arg_fetchatt) + + def do_COPY(self, tag, messages, mailbox, uid=0): + from twisted.internet import reactor + imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid) + deferLater(reactor, + 2, self.mbox.unset_recent_flags, messages) + deferLater(reactor, 1, self.mbox.signal_unread_to_ui) + + select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset, + imap4.IMAP4Server.arg_astring) + + def notifyNew(self, ignored): + """ + Notify new messages to listeners. + """ + self.mbox.notify_new() + + def _cbSelectWork(self, mbox, cmdName, tag): + """ + Callback for selectWork, patched to avoid conformance errors due to + incomplete UIDVALIDITY line. + """ + if mbox is None: + self.sendNegativeResponse(tag, 'No such mailbox') + return + if '\\noselect' in [s.lower() for s in mbox.getFlags()]: + self.sendNegativeResponse(tag, 'Mailbox cannot be selected') + return + + flags = mbox.getFlags() + self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') + self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') + self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) + + # Patched ------------------------------------------------------- + # imaptest was complaining about the incomplete line, we're adding + # "UIDs valid" here. + self.sendPositiveResponse( + None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity()) + # ---------------------------------------------------------------- + + s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY' + mbox.addListener(self) + self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName)) + self.state = 'select' + self.mbox = mbox + + def checkpoint(self): + """ + Called when the client issues a CHECK command. + + This should perform any checkpoint operations required by the server. + It may be a long running operation, but may not block. If it returns + a deferred, the client will only be informed of success (or failure) + when the deferred's callback (or errback) is invoked. + """ + # TODO return the output of _memstore.is_writing + # XXX and that should return a deferred! + return None -- cgit v1.2.3 From ff28e22977db802c87f0b7be99e37c6de29183e9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 23 Jan 2014 13:32:01 -0400 Subject: Unset new flag after successful write --- src/leap/mail/imap/server.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 8bd875b..c95a9be 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -196,4 +196,9 @@ class LeapIMAPServer(imap4.IMAP4Server): """ # TODO return the output of _memstore.is_writing # XXX and that should return a deferred! + + # XXX fake a delayed operation, to debug problem with messages getting + # back to the source mailbox... + import time + time.sleep(2) return None -- cgit v1.2.3 From f5365ae0c2edb8b3e879f876f2f7e42b25f4616a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 27 Jan 2014 16:11:53 -0400 Subject: handle last_uid property in memory store --- src/leap/mail/imap/server.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index c95a9be..3a6ac9a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -199,6 +199,7 @@ class LeapIMAPServer(imap4.IMAP4Server): # XXX fake a delayed operation, to debug problem with messages getting # back to the source mailbox... + print "faking checkpoint..." import time time.sleep(2) return None -- cgit v1.2.3 From a7e0054b595822325f749b0b1df7d25cab4e6486 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 28 Jan 2014 18:39:59 -0400 Subject: docstring fixes Also some fixes for None comparisons. --- src/leap/mail/imap/server.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 3a6ac9a..b77678a 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -20,6 +20,7 @@ Leap IMAP4 Server Implementation. from copy import copy from twisted import cred +from twisted.internet import defer from twisted.internet.defer import maybeDeferred from twisted.internet.task import deferLater from twisted.mail import imap4 @@ -132,6 +133,7 @@ class LeapIMAPServer(imap4.IMAP4Server): ).addErrback( ebFetch, tag) + # XXX should be a callback deferLater(reactor, 2, self.mbox.unset_recent_flags, messages) deferLater(reactor, 1, self.mbox.signal_unread_to_ui) @@ -139,12 +141,17 @@ class LeapIMAPServer(imap4.IMAP4Server): select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, imap4.IMAP4Server.arg_fetchatt) + def on_copy_finished(self, defers): + d = defer.gatherResults(filter(None, defers)) + d.addCallback(self.notifyNew) + d.addCallback(self.mbox.signal_unread_to_ui) + def do_COPY(self, tag, messages, mailbox, uid=0): from twisted.internet import reactor - imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid) - deferLater(reactor, - 2, self.mbox.unset_recent_flags, messages) - deferLater(reactor, 1, self.mbox.signal_unread_to_ui) + defers = [] + d = imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid) + defers.append(d) + deferLater(reactor, 0, self.on_copy_finished, defers) select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset, imap4.IMAP4Server.arg_astring) @@ -201,5 +208,5 @@ class LeapIMAPServer(imap4.IMAP4Server): # back to the source mailbox... print "faking checkpoint..." import time - time.sleep(2) + time.sleep(5) return None -- cgit v1.2.3 From 1b71ba510a2e6680f1ecc84eacfc492b0bbe24fc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 29 Jan 2014 00:54:20 -0400 Subject: Fix copy and deletion problems * reorganize and simplify STORE command processing * add the notification after the processing of the whole sequence --- src/leap/mail/imap/server.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index b77678a..7bca39d 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -99,10 +99,9 @@ class LeapIMAPServer(imap4.IMAP4Server): Overwritten fetch dispatcher to use the fast fetch_flags method """ - from twisted.internet import reactor if not query: self.sendPositiveResponse(tag, 'FETCH complete') - return # XXX ??? + return cbFetch = self._IMAP4Server__cbFetch ebFetch = self._IMAP4Server__ebFetch @@ -131,16 +130,19 @@ class LeapIMAPServer(imap4.IMAP4Server): ).addCallback( cbFetch, tag, query, uid ).addErrback( - ebFetch, tag) - - # XXX should be a callback - deferLater(reactor, - 2, self.mbox.unset_recent_flags, messages) - deferLater(reactor, 1, self.mbox.signal_unread_to_ui) + ebFetch, tag + ).addCallback( + self.on_fetch_finished, messages) select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, imap4.IMAP4Server.arg_fetchatt) + def on_fetch_finished(self, _, messages): + from twisted.internet import reactor + deferLater(reactor, 0, self.notifyNew) + deferLater(reactor, 0, self.mbox.unset_recent_flags, messages) + deferLater(reactor, 0, self.mbox.signal_unread_to_ui) + def on_copy_finished(self, defers): d = defer.gatherResults(filter(None, defers)) d.addCallback(self.notifyNew) @@ -156,7 +158,7 @@ class LeapIMAPServer(imap4.IMAP4Server): select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset, imap4.IMAP4Server.arg_astring) - def notifyNew(self, ignored): + def notifyNew(self, ignored=None): """ Notify new messages to listeners. """ @@ -203,10 +205,4 @@ class LeapIMAPServer(imap4.IMAP4Server): """ # TODO return the output of _memstore.is_writing # XXX and that should return a deferred! - - # XXX fake a delayed operation, to debug problem with messages getting - # back to the source mailbox... - print "faking checkpoint..." - import time - time.sleep(5) return None -- cgit v1.2.3 From 18fed49c4143eb764ae9e806882d24f8f4e95744 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 2 Feb 2014 09:26:37 -0400 Subject: fix missing content after in-memory add because THE KEYS WILL BE STRINGS AFTER ADDED TO SOLEDAD Can I remember that? * Fix copy from local folders * Fix copy when we already have a copy of the message in the inbox, marked as deleted. * Fix also bad deferred.succeed in add_msg when it already exist. --- src/leap/mail/imap/server.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 7bca39d..ba63846 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -139,14 +139,22 @@ class LeapIMAPServer(imap4.IMAP4Server): def on_fetch_finished(self, _, messages): from twisted.internet import reactor + + print "FETCH FINISHED -- NOTIFY NEW" deferLater(reactor, 0, self.notifyNew) deferLater(reactor, 0, self.mbox.unset_recent_flags, messages) deferLater(reactor, 0, self.mbox.signal_unread_to_ui) def on_copy_finished(self, defers): d = defer.gatherResults(filter(None, defers)) - d.addCallback(self.notifyNew) - d.addCallback(self.mbox.signal_unread_to_ui) + + def when_finished(result): + log.msg("COPY FINISHED") + self.notifyNew() + self.mbox.signal_unread_to_ui() + d.addCallback(when_finished) + #d.addCallback(self.notifyNew) + #d.addCallback(self.mbox.signal_unread_to_ui) def do_COPY(self, tag, messages, mailbox, uid=0): from twisted.internet import reactor @@ -162,6 +170,7 @@ class LeapIMAPServer(imap4.IMAP4Server): """ Notify new messages to listeners. """ + print "TRYING TO NOTIFY NEW" self.mbox.notify_new() def _cbSelectWork(self, mbox, cmdName, tag): -- cgit v1.2.3 From 26d6db6210eaca18002c1ec8c5619d7fbd3e4243 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 6 Feb 2014 10:29:18 -0400 Subject: temporarily nuke out the fetch_heders diversion --- src/leap/mail/imap/server.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index ba63846..f4b9f71 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -114,14 +114,16 @@ class LeapIMAPServer(imap4.IMAP4Server): ).addCallback( cbFetch, tag, query, uid ).addErrback(ebFetch, tag) - elif len(query) == 1 and str(query[0]) == "rfc822.header": - self._oldTimeout = self.setTimeout(None) + + # XXX not implemented yet --- should hit memstore + #elif len(query) == 1 and str(query[0]) == "rfc822.header": + #self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator - maybeDeferred( - self.mbox.fetch_headers, messages, uid=uid - ).addCallback( - cbFetch, tag, query, uid - ).addErrback(ebFetch, tag) + #maybeDeferred( + #self.mbox.fetch_headers, messages, uid=uid + #).addCallback( + #cbFetch, tag, query, uid + #).addErrback(ebFetch, tag) else: self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator -- cgit v1.2.3 From ee0786c57d72aa8b8da76533f33c3dd65253a878 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 6 Feb 2014 18:11:20 -0400 Subject: long-due update to unittests! So we're safe under the green lights before further rewriting. :) --- src/leap/mail/imap/server.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index f4b9f71..89fb46d 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -41,6 +41,7 @@ class LeapIMAPServer(imap4.IMAP4Server): soledad = kwargs.pop('soledad', None) uuid = kwargs.pop('uuid', None) userid = kwargs.pop('userid', None) + leap_assert(soledad, "need a soledad instance") leap_assert_type(soledad, Soledad) leap_assert(uuid, "need a user in the initialization") -- cgit v1.2.3 From a05a00a78d750904456a649334bba619b07a3380 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 01:40:30 -0400 Subject: call notify in reactor --- src/leap/mail/imap/server.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 89fb46d..3497a8b 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -56,6 +56,9 @@ class LeapIMAPServer(imap4.IMAP4Server): # populate the test account properly (and only once # per session) + from twisted.internet import reactor + self.reactor = reactor + def lineReceived(self, line): """ Attempt to parse a single line from the server. @@ -141,30 +144,23 @@ class LeapIMAPServer(imap4.IMAP4Server): imap4.IMAP4Server.arg_fetchatt) def on_fetch_finished(self, _, messages): - from twisted.internet import reactor - - print "FETCH FINISHED -- NOTIFY NEW" - deferLater(reactor, 0, self.notifyNew) - deferLater(reactor, 0, self.mbox.unset_recent_flags, messages) - deferLater(reactor, 0, self.mbox.signal_unread_to_ui) + deferLater(self.reactor, 0, self.notifyNew) + deferLater(self.reactor, 0, self.mbox.unset_recent_flags, messages) + deferLater(self.reactor, 0, self.mbox.signal_unread_to_ui) def on_copy_finished(self, defers): d = defer.gatherResults(filter(None, defers)) def when_finished(result): - log.msg("COPY FINISHED") self.notifyNew() self.mbox.signal_unread_to_ui() d.addCallback(when_finished) - #d.addCallback(self.notifyNew) - #d.addCallback(self.mbox.signal_unread_to_ui) def do_COPY(self, tag, messages, mailbox, uid=0): - from twisted.internet import reactor defers = [] d = imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid) defers.append(d) - deferLater(reactor, 0, self.on_copy_finished, defers) + deferLater(self.reactor, 0, self.on_copy_finished, defers) select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset, imap4.IMAP4Server.arg_astring) @@ -173,8 +169,7 @@ class LeapIMAPServer(imap4.IMAP4Server): """ Notify new messages to listeners. """ - print "TRYING TO NOTIFY NEW" - self.mbox.notify_new() + self.reactor.callFromThread(self.mbox.notify_new) def _cbSelectWork(self, mbox, cmdName, tag): """ -- cgit v1.2.3 From fd9c8c2e3c88476b90805b689f6914fe5eac16df Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 02:53:28 -0400 Subject: defer fetch to thread also, dispatch query for all headers to its own method. --- src/leap/mail/imap/server.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 3497a8b..7c09784 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -119,15 +119,14 @@ class LeapIMAPServer(imap4.IMAP4Server): cbFetch, tag, query, uid ).addErrback(ebFetch, tag) - # XXX not implemented yet --- should hit memstore - #elif len(query) == 1 and str(query[0]) == "rfc822.header": - #self._oldTimeout = self.setTimeout(None) + elif len(query) == 1 and str(query[0]) == "rfc822.header": + self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator - #maybeDeferred( - #self.mbox.fetch_headers, messages, uid=uid - #).addCallback( - #cbFetch, tag, query, uid - #).addErrback(ebFetch, tag) + maybeDeferred( + self.mbox.fetch_headers, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) else: self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator -- cgit v1.2.3 From 985ff0a78a8df0eafb7789383f711b9e5ceb1cb6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 17 Feb 2014 11:08:37 -0400 Subject: Remove notify_new callbacks from fetch and copy. This fixes a bug with qtreactor that was making the 'OK foo copied' not being delivered. This or something similar will probably have to be re-added, because on the current state the destination folder will not receive the notification if it's selected *before* the copy operation has finished. But in this way we have a clean slate that is working properly. The bottleneck in the copy/append operations seems to have moved to the select operation now. --- src/leap/mail/imap/server.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 7c09784..5da9bfd 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -20,9 +20,7 @@ Leap IMAP4 Server Implementation. from copy import copy from twisted import cred -from twisted.internet import defer from twisted.internet.defer import maybeDeferred -from twisted.internet.task import deferLater from twisted.mail import imap4 from twisted.python import log @@ -135,35 +133,11 @@ class LeapIMAPServer(imap4.IMAP4Server): ).addCallback( cbFetch, tag, query, uid ).addErrback( - ebFetch, tag - ).addCallback( - self.on_fetch_finished, messages) + ebFetch, tag) select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, imap4.IMAP4Server.arg_fetchatt) - def on_fetch_finished(self, _, messages): - deferLater(self.reactor, 0, self.notifyNew) - deferLater(self.reactor, 0, self.mbox.unset_recent_flags, messages) - deferLater(self.reactor, 0, self.mbox.signal_unread_to_ui) - - def on_copy_finished(self, defers): - d = defer.gatherResults(filter(None, defers)) - - def when_finished(result): - self.notifyNew() - self.mbox.signal_unread_to_ui() - d.addCallback(when_finished) - - def do_COPY(self, tag, messages, mailbox, uid=0): - defers = [] - d = imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid) - defers.append(d) - deferLater(self.reactor, 0, self.on_copy_finished, defers) - - select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset, - imap4.IMAP4Server.arg_astring) - def notifyNew(self, ignored=None): """ Notify new messages to listeners. -- cgit v1.2.3 From 3f69d4216c731c2985bb5d09127c58e196a29006 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 26 Feb 2014 10:46:44 -0400 Subject: Implement non-synchronizing literals (rfc2088) Closes: #5190 This also paves the way to MULTIAPPEND IMAP Extension (rfc3502) Related to: Feature #5182 --- src/leap/mail/imap/server.py | 187 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) (limited to 'src/leap/mail/imap/server.py') diff --git a/src/leap/mail/imap/server.py b/src/leap/mail/imap/server.py index 5da9bfd..fe56ea6 100644 --- a/src/leap/mail/imap/server.py +++ b/src/leap/mail/imap/server.py @@ -29,6 +29,11 @@ from leap.common.check import leap_assert, leap_assert_type from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN from leap.soledad.client import Soledad +# imports for LITERAL+ patch +from twisted.internet import defer, interfaces +from twisted.mail.imap4 import IllegalClientResponse +from twisted.mail.imap4 import LiteralString, LiteralFile + class LeapIMAPServer(imap4.IMAP4Server): """ @@ -186,3 +191,185 @@ class LeapIMAPServer(imap4.IMAP4Server): # TODO return the output of _memstore.is_writing # XXX and that should return a deferred! return None + + ############################################################# + # + # Twisted imap4 patch to support LITERAL+ extension + # TODO send this patch upstream asap! + # + ############################################################# + + def capabilities(self): + cap = {'AUTH': self.challengers.keys()} + if self.ctx and self.canStartTLS: + t = self.transport + ti = interfaces.ISSLTransport + if not self.startedTLS and ti(t, None) is None: + cap['LOGINDISABLED'] = None + cap['STARTTLS'] = None + cap['NAMESPACE'] = None + cap['IDLE'] = None + # patched ############ + cap['LITERAL+'] = None + ###################### + return cap + + def _stringLiteral(self, size, literal_plus=False): + if size > self._literalStringLimit: + raise IllegalClientResponse( + "Literal too long! I accept at most %d octets" % + (self._literalStringLimit,)) + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralString(size, d) + # Patched ########################################################### + if not literal_plus: + self.sendContinuationRequest('Ready for %d octets of text' % size) + ##################################################################### + self.setRawMode() + return d + + def _fileLiteral(self, size, literal_plus=False): + d = defer.Deferred() + self.parseState = 'pending' + self._pendingLiteral = LiteralFile(size, d) + if not literal_plus: + self.sendContinuationRequest('Ready for %d octets of data' % size) + self.setRawMode() + return d + + def arg_astring(self, line): + """ + Parse an astring from the line, return (arg, rest), possibly + via a deferred (to handle literals) + """ + line = line.strip() + if not line: + raise IllegalClientResponse("Missing argument") + d = None + arg, rest = None, None + if line[0] == '"': + try: + spam, arg, rest = line.split('"', 2) + rest = rest[1:] # Strip space + except ValueError: + raise IllegalClientResponse("Unmatched quotes") + elif line[0] == '{': + # literal + if line[-1] != '}': + raise IllegalClientResponse("Malformed literal") + + # Patched ################ + if line[-2] == "+": + literalPlus = True + size_end = -2 + else: + literalPlus = False + size_end = -1 + + try: + size = int(line[1:size_end]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: " + line[1:size_end]) + d = self._stringLiteral(size, literalPlus) + ########################## + else: + arg = line.split(' ', 1) + if len(arg) == 1: + arg.append('') + arg, rest = arg + return d or (arg, rest) + + def arg_literal(self, line): + """ + Parse a literal from the line + """ + if not line: + raise IllegalClientResponse("Missing argument") + + if line[0] != '{': + raise IllegalClientResponse("Missing literal") + + if line[-1] != '}': + raise IllegalClientResponse("Malformed literal") + + # Patched ################## + if line[-2] == "+": + literalPlus = True + size_end = -2 + else: + literalPlus = False + size_end = -1 + + try: + size = int(line[1:size_end]) + except ValueError: + raise IllegalClientResponse( + "Bad literal size: " + line[1:size_end]) + + return self._fileLiteral(size, literalPlus) + ############################# + + # Need to override the command table after patching + # arg_astring and arg_literal + + do_LOGIN = imap4.IMAP4Server.do_LOGIN + do_CREATE = imap4.IMAP4Server.do_CREATE + do_DELETE = imap4.IMAP4Server.do_DELETE + do_RENAME = imap4.IMAP4Server.do_RENAME + do_SUBSCRIBE = imap4.IMAP4Server.do_SUBSCRIBE + do_UNSUBSCRIBE = imap4.IMAP4Server.do_UNSUBSCRIBE + do_STATUS = imap4.IMAP4Server.do_STATUS + do_APPEND = imap4.IMAP4Server.do_APPEND + do_COPY = imap4.IMAP4Server.do_COPY + + _selectWork = imap4.IMAP4Server._selectWork + _listWork = imap4.IMAP4Server._listWork + arg_plist = imap4.IMAP4Server.arg_plist + arg_seqset = imap4.IMAP4Server.arg_seqset + opt_plist = imap4.IMAP4Server.opt_plist + opt_datetime = imap4.IMAP4Server.opt_datetime + + unauth_LOGIN = (do_LOGIN, arg_astring, arg_astring) + + auth_SELECT = (_selectWork, arg_astring, 1, 'SELECT') + select_SELECT = auth_SELECT + + auth_CREATE = (do_CREATE, arg_astring) + select_CREATE = auth_CREATE + + auth_EXAMINE = (_selectWork, arg_astring, 0, 'EXAMINE') + select_EXAMINE = auth_EXAMINE + + auth_DELETE = (do_DELETE, arg_astring) + select_DELETE = auth_DELETE + + auth_RENAME = (do_RENAME, arg_astring, arg_astring) + select_RENAME = auth_RENAME + + auth_SUBSCRIBE = (do_SUBSCRIBE, arg_astring) + select_SUBSCRIBE = auth_SUBSCRIBE + + auth_UNSUBSCRIBE = (do_UNSUBSCRIBE, arg_astring) + select_UNSUBSCRIBE = auth_UNSUBSCRIBE + + auth_LIST = (_listWork, arg_astring, arg_astring, 0, 'LIST') + select_LIST = auth_LIST + + auth_LSUB = (_listWork, arg_astring, arg_astring, 1, 'LSUB') + select_LSUB = auth_LSUB + + auth_STATUS = (do_STATUS, arg_astring, arg_plist) + select_STATUS = auth_STATUS + + auth_APPEND = (do_APPEND, arg_astring, opt_plist, opt_datetime, + arg_literal) + select_APPEND = auth_APPEND + + select_COPY = (do_COPY, arg_seqset, arg_astring) + + + ############################################################# + # END of Twisted imap4 patch to support LITERAL+ extension + ############################################################# -- cgit v1.2.3