diff options
Diffstat (limited to 'src/leap/mail/imap/mailbox.py')
| -rw-r--r-- | src/leap/mail/imap/mailbox.py | 131 |
1 files changed, 58 insertions, 73 deletions
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py index 108d0da..b5c5719 100644 --- a/src/leap/mail/imap/mailbox.py +++ b/src/leap/mail/imap/mailbox.py @@ -26,7 +26,7 @@ import cStringIO from collections import defaultdict from twisted.internet import defer -from twisted.internet.task import deferLater +#from twisted.internet.task import deferLater from twisted.python import log from twisted.mail import imap4 @@ -119,6 +119,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if not self.getFlags(): self.setFlags(self.INIT_FLAGS) + if self._memstore: + self.prime_last_uid_to_memstore() + @property def listeners(self): """ @@ -132,6 +135,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ return self._listeners[self.mbox] + # TODO this grows too crazily when many instances are fired, like + # during imaptest stress testing. Should have a queue of limited size + # instead. def addListener(self, listener): """ Add a listener to the listeners queue. @@ -153,6 +159,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): """ self.listeners.remove(listener) + # TODO move completely to soledadstore, under memstore reponsibility. def _get_mbox(self): """ Return mailbox document. @@ -228,52 +235,28 @@ class SoledadMailbox(WithMsgFields, MBoxParser): def _get_last_uid(self): """ Return the last uid for this mailbox. + If we have a memory store, the last UID will be the highest + recorded UID in the message store, or a counter cached from + the mailbox document in soledad if this is higher. :return: the last uid for messages in this mailbox :rtype: bool """ - mbox = self._get_mbox() - if not mbox: - logger.error("We could not get a mbox!") - # XXX It looks like it has been corrupted. - # We need to be able to survive this. - return None - last = mbox.content.get(self.LAST_UID_KEY, 1) - if self._memstore: - last = max(last, self._memstore.get_last_uid(mbox)) + last = self._memstore.get_last_uid(self.mbox) + print "last uid for %s: %s (from memstore)" % (self.mbox, last) return last - def _set_last_uid(self, uid): - """ - Sets the last uid for this mailbox. + last_uid = property( + _get_last_uid, doc="Last_UID attribute.") - :param uid: the uid to be set - :type uid: int + def prime_last_uid_to_memstore(self): """ - leap_assert(isinstance(uid, int), "uid has to be int") - mbox = self._get_mbox() - key = self.LAST_UID_KEY - - count = self.getMessageCount() - - # XXX safety-catch. If we do get duplicates, - # we want to avoid further duplication. - - if uid >= count: - value = uid - else: - # something is wrong, - # just set the last uid - # beyond the max msg count. - logger.debug("WRONG uid < count. Setting last uid to %s", count) - value = count - - mbox.content[key] = value - # XXX this should be set in the memorystore instead!!! - self._soledad.put_doc(mbox) - - last_uid = property( - _get_last_uid, _set_last_uid, doc="Last_UID attribute.") + Prime memstore with last_uid value + """ + set_exist = set(self.messages.all_uid_iter()) + last = max(set_exist) + 1 if set_exist else 1 + logger.info("Priming Soledad last_uid to %s" % (last,)) + self._memstore.set_last_soledad_uid(self.mbox, last) def getUIDValidity(self): """ @@ -315,8 +298,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :rtype: int """ with self.next_uid_lock: - self.last_uid += 1 - return self.last_uid + if self._memstore: + return self.last_uid + 1 + else: + # XXX after lock, it should be safe to + # return just the increment here, and + # have a different method that actually increments + # the counter when really adding. + self.last_uid += 1 + return self.last_uid def getMessageCount(self): """ @@ -397,26 +387,26 @@ class SoledadMailbox(WithMsgFields, MBoxParser): :return: a deferred that evals to None """ + # TODO have a look at the cases for internal date in the rfc if isinstance(message, (cStringIO.OutputType, StringIO.StringIO)): message = message.getvalue() - # XXX we should treat the message as an IMessage from here + + # XXX we could treat the message as an IMessage from here leap_assert_type(message, basestring) - uid_next = self.getUIDNext() - logger.debug('Adding msg with UID :%s' % uid_next) if flags is None: flags = tuple() else: flags = tuple(str(flag) for flag in flags) - d = self._do_add_message(message, flags=flags, date=date, uid=uid_next) + d = self._do_add_message(message, flags=flags, date=date) return d - def _do_add_message(self, message, flags, date, uid): + def _do_add_message(self, message, flags, date): """ - Calls to the messageCollection add_msg method (deferred to thread). + Calls to the messageCollection add_msg method. Invoked from addMessage. """ - d = self.messages.add_msg(message, flags=flags, date=date, uid=uid) + d = self.messages.add_msg(message, flags=flags, date=date) # XXX Removing notify temporarily. # This is interfering with imaptest results. I'm not clear if it's # because we clutter the logging or because the set of listeners is @@ -456,6 +446,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # XXX removing the mailbox in situ for now, # we should postpone the removal + + # XXX move to memory store?? self._soledad.delete_doc(self._get_mbox()) def _close_cb(self, result): @@ -466,8 +458,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): Expunge and mark as closed """ d = self.expunge() - d.addCallback(self._close_cb) - return d + #d.addCallback(self._close_cb) + #return d def _expunge_cb(self, result): return result @@ -479,22 +471,15 @@ class SoledadMailbox(WithMsgFields, MBoxParser): print "EXPUNGE!" if not self.isWriteable(): raise imap4.ReadOnlyMailbox - mstore = self._memstore - if mstore is not None: - deleted = mstore.all_deleted_uid_iter(self.mbox) - print "deleted ", list(deleted) - for uid in deleted: - mstore.remove_message(self.mbox, uid) - - print "now deleting from soledad" - d = self.messages.remove_all_deleted() - d.addCallback(self._expunge_cb) - d.addCallback(self.messages.reset_last_uid) - - # XXX DEBUG ------------------- - # FIXME !!! - # XXX should remove the hdocset too!!! - return d + + return self._memstore.expunge(self.mbox) + + # TODO we can defer this back when it's correct + # but we should make sure the memstore has been synced. + + #d = self._memstore.expunge(self.mbox) + #d.addCallback(self._expunge_cb) + #return d def _bound_seq(self, messages_asked): """ @@ -783,12 +768,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): # IMessageCopier @deferred + #@profile def copy(self, messageObject): """ Copy the given message object into this mailbox. """ from twisted.internet import reactor - uid_next = self.getUIDNext() msg = messageObject memstore = self._memstore @@ -796,7 +781,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser): fdoc = msg._fdoc hdoc = msg._hdoc if not fdoc: - logger.debug("Tried to copy a MSG with no fdoc") + logger.warning("Tried to copy a MSG with no fdoc") return new_fdoc = copy.deepcopy(fdoc.content) @@ -807,11 +792,12 @@ class SoledadMailbox(WithMsgFields, MBoxParser): if exist: print "Destination message already exists!" - else: print "DO COPY MESSAGE!" + mbox = self.mbox + uid_next = memstore.increment_last_soledad_uid(mbox) new_fdoc[self.UID_KEY] = uid_next - new_fdoc[self.MBOX_KEY] = self.mbox + new_fdoc[self.MBOX_KEY] = mbox # XXX set recent! @@ -824,9 +810,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser): self._memstore.create_message( self.mbox, uid_next, MessageWrapper( - new_fdoc, hdoc.content)) - - deferLater(reactor, 1, self.notify_new) + new_fdoc, hdoc.content), + notify_on_disk=False) # convenience fun |