summaryrefslogtreecommitdiff
path: root/src/leap/mail/imap
diff options
context:
space:
mode:
authorKali Kaneko <kali@leap.se>2014-11-25 15:04:26 +0100
committerKali Kaneko <kali@leap.se>2015-02-11 14:05:42 -0400
commit6ede495b94501a4cbdfd985dcdf4be4f582bbb9b (patch)
tree8afc3622e5afe865285ec25a4da85b0f1a14ecb5 /src/leap/mail/imap
parentea82f75f5465de47c4a838fbd1dfe8b2030fd842 (diff)
Serializable Models + Soledad Adaptor
Diffstat (limited to 'src/leap/mail/imap')
-rw-r--r--src/leap/mail/imap/account.py223
-rw-r--r--src/leap/mail/imap/fields.py132
-rw-r--r--src/leap/mail/imap/index.py90
-rw-r--r--src/leap/mail/imap/interfaces.py2
-rw-r--r--src/leap/mail/imap/mailbox.py76
-rw-r--r--src/leap/mail/imap/messages.py484
-rw-r--r--src/leap/mail/imap/parser.py45
-rw-r--r--src/leap/mail/imap/tests/test_imap.py2
-rw-r--r--src/leap/mail/imap/tests/utils.py15
9 files changed, 301 insertions, 768 deletions
diff --git a/src/leap/mail/imap/account.py b/src/leap/mail/imap/account.py
index fe466cb..7dfbbd1 100644
--- a/src/leap/mail/imap/account.py
+++ b/src/leap/mail/imap/account.py
@@ -28,10 +28,10 @@ from twisted.python import log
from zope.interface import implements
from leap.common.check import leap_assert, leap_assert_type
-from leap.mail.imap.index import IndexedDB
+
+from leap.mail.mail import Account
from leap.mail.imap.fields import WithMsgFields
-from leap.mail.imap.parser import MBoxParser
-from leap.mail.imap.mailbox import SoledadMailbox
+from leap.mail.imap.mailbox import SoledadMailbox, normalize_mailbox
from leap.soledad.client import Soledad
logger = logging.getLogger(__name__)
@@ -39,7 +39,6 @@ logger = logging.getLogger(__name__)
PROFILE_CMD = os.environ.get('LEAP_PROFILE_IMAPCMD', False)
if PROFILE_CMD:
-
def _debugProfiling(result, cmdname, start):
took = (time.time() - start) * 1000
log.msg("CMD " + cmdname + " TOOK: " + str(took) + " msec")
@@ -47,96 +46,43 @@ if PROFILE_CMD:
#######################################
-# Soledad Account
+# Soledad IMAP Account
#######################################
+# TODO remove MsgFields too
-# TODO change name to LeapIMAPAccount, since we're using
-# the memstore.
-# IndexedDB should also not be here anymore.
-
-class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
+class IMAPAccount(WithMsgFields):
"""
- An implementation of IAccount and INamespacePresenteer
+ An implementation of an imap4 Account
that is backed by Soledad Encrypted Documents.
"""
implements(imap4.IAccount, imap4.INamespacePresenter)
- _soledad = None
selected = None
closed = False
- _initialized = False
- def __init__(self, account_name, soledad, memstore=None):
+ def __init__(self, user_id, store):
"""
- Creates a SoledadAccountIndex that keeps track of the mailboxes
- and subscriptions handled by this account.
+ Keeps track of the mailboxes and subscriptions handled by this account.
- :param acct_name: The name of the account (user id).
- :type acct_name: str
+ :param account: The name of the account (user id).
+ :type account: str
- :param soledad: a Soledad instance.
- :type soledad: Soledad
- :param memstore: a MemoryStore instance.
- :type memstore: MemoryStore
+ :param store: a Soledad instance.
+ :type store: Soledad
"""
- leap_assert(soledad, "Need a soledad instance to initialize")
- leap_assert_type(soledad, Soledad)
+ # XXX assert a generic store interface instead, so that we
+ # can plug the memory store wrapper seamlessly.
+ leap_assert(store, "Need a store instance to initialize")
+ leap_assert_type(store, Soledad)
# XXX SHOULD assert too that the name matches the user/uuid with which
# soledad has been initialized.
+ self.user_id = user_id
+ self.account = Account(store)
- # XXX ??? why is this parsing mailbox name??? it's account...
- # userid? homogenize.
- self._account_name = self._parse_mailbox_name(account_name)
- self._soledad = soledad
- self._memstore = memstore
-
- self.__mailboxes = set([])
-
- self._deferred_initialization = defer.Deferred()
- self._initialize_storage()
-
- def _initialize_storage(self):
-
- def add_mailbox_if_none(result):
- # every user should have the right to an inbox folder
- # at least, so let's make one!
- if not self.mailboxes:
- self.addMailbox(self.INBOX_NAME)
-
- def finish_initialization(result):
- self._initialized = True
- self._deferred_initialization.callback(None)
-
- def load_mbox_cache(result):
- d = self._load_mailboxes()
- d.addCallback(lambda _: result)
- return d
-
- d = self.initialize_db()
-
- d.addCallback(load_mbox_cache)
- d.addCallback(add_mailbox_if_none)
- d.addCallback(finish_initialization)
-
- def callWhenReady(self, cb):
- if self._initialized:
- cb(self)
- return defer.succeed(None)
- else:
- self._deferred_initialization.addCallback(cb)
- return self._deferred_initialization
-
- def _get_empty_mailbox(self):
- """
- Returns an empty mailbox.
-
- :rtype: dict
- """
- return copy.deepcopy(self.EMPTY_MBOX)
-
+ # XXX should hide this in the adaptor...
def _get_mailbox_by_name(self, name):
"""
Return an mbox document by name.
@@ -146,32 +92,17 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:rtype: SoledadDocument
"""
- # XXX use soledadstore instead ...;
def get_first_if_any(docs):
return docs[0] if docs else None
- d = self._soledad.get_from_index(
+ d = self._store.get_from_index(
self.TYPE_MBOX_IDX, self.MBOX_KEY,
- self._parse_mailbox_name(name))
+ normalize_mailbox(name))
d.addCallback(get_first_if_any)
return d
- @property
- def mailboxes(self):
- """
- A list of the current mailboxes for this account.
- :rtype: set
- """
- return sorted(self.__mailboxes)
-
- def _load_mailboxes(self):
- def update_mailboxes(db_indexes):
- self.__mailboxes.update(
- [doc.content[self.MBOX_KEY] for doc in db_indexes])
- d = self._soledad.get_from_index(self.TYPE_IDX, self.MBOX_KEY)
- d.addCallback(update_mailboxes)
- return d
-
+ # XXX move to Account?
+ # XXX needed?
def getMailbox(self, name):
"""
Return a Mailbox with that name, without selecting it.
@@ -182,18 +113,28 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:returns: a a SoledadMailbox instance
:rtype: SoledadMailbox
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
- if name not in self.mailboxes:
+ if name not in self.account.mailboxes:
raise imap4.MailboxException("No such mailbox: %r" % name)
- return SoledadMailbox(name, self._soledad,
- memstore=self._memstore)
+ # XXX Does mailbox really need reference to soledad?
+ return SoledadMailbox(name, self._store)
#
# IAccount
#
+ def _get_empty_mailbox(self):
+ """
+ Returns an empty mailbox.
+
+ :rtype: dict
+ """
+ # XXX move to mailbox module
+ return copy.deepcopy(mailbox.EMPTY_MBOX)
+
+ # TODO use mail.Account.add_mailbox
def addMailbox(self, name, creation_ts=None):
"""
Add a mailbox to the account.
@@ -209,7 +150,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:returns: a Deferred that will contain the document if successful.
:rtype: bool
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
leap_assert(name, "Need a mailbox name to create a mailbox")
@@ -232,10 +173,12 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
d.addCallback(lambda _: result)
return d
- d = self._soledad.create_doc(mbox)
+ d = self._store.create_doc(mbox)
d.addCallback(load_mbox_cache)
return d
+ # TODO use mail.Account.create_mailbox?
+ # Watch out, imap specific exceptions raised here.
def create(self, pathspec):
"""
Create a new mailbox from the given hierarchical name.
@@ -254,9 +197,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:raise MailboxException: Raised if this mailbox cannot be added.
"""
# TODO raise MailboxException
- paths = filter(
- None,
- self._parse_mailbox_name(pathspec).split('/'))
+ paths = filter(None, normalize_mailbox(pathspec).split('/'))
subs = []
sep = '/'
@@ -295,6 +236,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
d1.addCallback(load_mbox_cache)
return d1
+ # TODO use mail.Account.get_collection_by_mailbox
def select(self, name, readwrite=1):
"""
Selects a mailbox.
@@ -307,21 +249,16 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:rtype: SoledadMailbox
"""
- if PROFILE_CMD:
- start = time.time()
-
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
if name not in self.mailboxes:
logger.warning("No such mailbox!")
return None
self.selected = name
- sm = SoledadMailbox(
- name, self._soledad, self._memstore, readwrite)
- if PROFILE_CMD:
- _debugProfiling(None, "SELECT", start)
+ sm = SoledadMailbox(name, self._store, readwrite)
return sm
+ # TODO use mail.Account.delete_mailbox
def delete(self, name, force=False):
"""
Deletes a mailbox.
@@ -338,7 +275,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:type force: bool
:rtype: Deferred
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
if name not in self.mailboxes:
err = imap4.MailboxException("No such mailbox: %r" % name)
@@ -369,6 +306,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
# ??! -- can this be rite?
# self._index.removeMailbox(name)
+ # TODO use mail.Account.rename_mailbox
def rename(self, oldname, newname):
"""
Renames a mailbox.
@@ -379,8 +317,8 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:param newname: new name of the mailbox
:type newname: str
"""
- oldname = self._parse_mailbox_name(oldname)
- newname = self._parse_mailbox_name(newname)
+ oldname = normalize_mailbox(oldname)
+ newname = normalize_mailbox(newname)
if oldname not in self.mailboxes:
raise imap4.NoSuchMailbox(repr(oldname))
@@ -431,6 +369,32 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
inferiors.append(infname)
return inferiors
+ # TODO use mail.Account.list_mailboxes
+ def listMailboxes(self, ref, wildcard):
+ """
+ List the mailboxes.
+
+ from rfc 3501:
+ returns a subset of names from the complete set
+ of all names available to the client. Zero or more untagged LIST
+ replies are returned, containing the name attributes, hierarchy
+ delimiter, and name.
+
+ :param ref: reference name
+ :type ref: str
+
+ :param wildcard: mailbox name with possible wildcards
+ :type wildcard: str
+ """
+ # XXX use wildcard in index query
+ ref = self._inferiorNames(normalize_mailbox(ref))
+ wildcard = imap4.wildcardToRegexp(wildcard, '/')
+ return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
+
+ #
+ # The rest of the methods are specific for leap.mail.imap.account.Account
+ #
+
# TODO ------------------ can we preserve the attr?
# maybe add to memory store.
def isSubscribed(self, name):
@@ -442,6 +406,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:rtype: Deferred (will fire with bool)
"""
+ # TODO use Flags class
subscribed = self.SUBSCRIBED_KEY
def is_subscribed(mbox):
@@ -465,7 +430,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
def get_docs_content(docs):
return [doc.content[self.MBOX_KEY] for doc in docs]
- d = self._soledad.get_from_index(
+ d = self._store.get_from_index(
self.TYPE_SUBS_IDX, self.MBOX_KEY, '1')
d.addCallback(get_docs_content)
return d
@@ -488,7 +453,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
def update_subscribed_value(mbox):
mbox.content[subscribed] = value
- return self._soledad.put_doc(mbox)
+ return self._store.put_doc(mbox)
# maybe we should store subscriptions in another
# document...
@@ -508,7 +473,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:type name: str
:rtype: Deferred
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
def check_and_subscribe(subscriptions):
if name not in subscriptions:
@@ -525,7 +490,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
:type name: str
:rtype: Deferred
"""
- name = self._parse_mailbox_name(name)
+ name = normalize_mailbox(name)
def check_and_unsubscribe(subscriptions):
if name not in subscriptions:
@@ -539,28 +504,6 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
def getSubscriptions(self):
return self._get_subscriptions()
- def listMailboxes(self, ref, wildcard):
- """
- List the mailboxes.
-
- from rfc 3501:
- returns a subset of names from the complete set
- of all names available to the client. Zero or more untagged LIST
- replies are returned, containing the name attributes, hierarchy
- delimiter, and name.
-
- :param ref: reference name
- :type ref: str
-
- :param wildcard: mailbox name with possible wildcards
- :type wildcard: str
- """
- # XXX use wildcard in index query
- ref = self._inferiorNames(
- self._parse_mailbox_name(ref))
- wildcard = imap4.wildcardToRegexp(wildcard, '/')
- return [(i, self.getMailbox(i)) for i in ref if wildcard.match(i)]
-
#
# INamespacePresenter
#
@@ -592,4 +535,4 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB, MBoxParser):
"""
Representation string for this object.
"""
- return "<SoledadBackedAccount (%s)>" % self._account_name
+ return "<IMAPAccount (%s)>" % self.user_id
diff --git a/src/leap/mail/imap/fields.py b/src/leap/mail/imap/fields.py
index 4576939..a751c6d 100644
--- a/src/leap/mail/imap/fields.py
+++ b/src/leap/mail/imap/fields.py
@@ -17,7 +17,9 @@
"""
Fields for Mailbox and Message.
"""
-from leap.mail.imap.parser import MBoxParser
+
+# TODO deprecate !!! (move all to constants maybe?)
+# Flags -> foo
class WithMsgFields(object):
@@ -25,55 +27,12 @@ class WithMsgFields(object):
Container class for class-attributes to be shared by
several message-related classes.
"""
- # indexing
- CONTENT_HASH_KEY = "chash"
- PAYLOAD_HASH_KEY = "phash"
-
- # Internal representation of Message
-
- # flags doc
- UID_KEY = "uid"
- MBOX_KEY = "mbox"
- SEEN_KEY = "seen"
- DEL_KEY = "deleted"
- RECENT_KEY = "recent"
- FLAGS_KEY = "flags"
- MULTIPART_KEY = "multi"
- SIZE_KEY = "size"
-
- # headers
- HEADERS_KEY = "headers"
- DATE_KEY = "date"
- SUBJECT_KEY = "subject"
- PARTS_MAP_KEY = "part_map"
- BODY_KEY = "body" # link to phash of body
- MSGID_KEY = "msgid"
-
- # content
- LINKED_FROM_KEY = "lkf" # XXX not implemented yet!
- RAW_KEY = "raw"
- CTYPE_KEY = "ctype"
-
# Mailbox specific keys
- CLOSED_KEY = "closed"
- CREATED_KEY = "created"
- SUBSCRIBED_KEY = "subscribed"
- RW_KEY = "rw"
- LAST_UID_KEY = "lastuid"
+ CREATED_KEY = "created" # used???
+
RECENTFLAGS_KEY = "rct"
HDOCS_SET_KEY = "hdocset"
- # Document Type, for indexing
- TYPE_KEY = "type"
- TYPE_MBOX_VAL = "mbox"
- TYPE_FLAGS_VAL = "flags"
- TYPE_HEADERS_VAL = "head"
- TYPE_CONTENT_VAL = "cnt"
- TYPE_RECENT_VAL = "rct"
- TYPE_HDOCS_SET_VAL = "hdocset"
-
- INBOX_VAL = "inbox"
-
# Flags in Mailbox and Message
SEEN_FLAG = "\\Seen"
RECENT_FLAG = "\\Recent"
@@ -88,86 +47,5 @@ class WithMsgFields(object):
SUBJECT_FIELD = "Subject"
DATE_FIELD = "Date"
- # Index types
- # --------------
-
- TYPE_IDX = 'by-type'
- TYPE_MBOX_IDX = 'by-type-and-mbox'
- TYPE_MBOX_UID_IDX = 'by-type-and-mbox-and-uid'
- TYPE_SUBS_IDX = 'by-type-and-subscribed'
- TYPE_MSGID_IDX = 'by-type-and-message-id'
- TYPE_MBOX_SEEN_IDX = 'by-type-and-mbox-and-seen'
- TYPE_MBOX_RECT_IDX = 'by-type-and-mbox-and-recent'
- TYPE_MBOX_DEL_IDX = 'by-type-and-mbox-and-deleted'
- TYPE_MBOX_C_HASH_IDX = 'by-type-and-mbox-and-contenthash'
- TYPE_C_HASH_IDX = 'by-type-and-contenthash'
- TYPE_C_HASH_PART_IDX = 'by-type-and-contenthash-and-partnumber'
- TYPE_P_HASH_IDX = 'by-type-and-payloadhash'
-
- # Tomas created the `recent and seen index`, but the semantic is not too
- # correct since the recent flag is volatile.
- TYPE_MBOX_RECT_SEEN_IDX = 'by-type-and-mbox-and-recent-and-seen'
-
- # Soledad index for incoming mail, without decrypting errors.
- JUST_MAIL_IDX = "just-mail"
- # XXX the backward-compatible index, will be deprecated at 0.7
- JUST_MAIL_COMPAT_IDX = "just-mail-compat"
-
- INCOMING_KEY = "incoming"
- ERROR_DECRYPTING_KEY = "errdecr"
-
- KTYPE = TYPE_KEY
- MBOX_VAL = TYPE_MBOX_VAL
- CHASH_VAL = CONTENT_HASH_KEY
- PHASH_VAL = PAYLOAD_HASH_KEY
-
- INDEXES = {
- # generic
- TYPE_IDX: [KTYPE],
- TYPE_MBOX_IDX: [KTYPE, MBOX_VAL],
- TYPE_MBOX_UID_IDX: [KTYPE, MBOX_VAL, UID_KEY],
-
- # mailboxes
- TYPE_SUBS_IDX: [KTYPE, 'bool(subscribed)'],
-
- # fdocs uniqueness
- TYPE_MBOX_C_HASH_IDX: [KTYPE, MBOX_VAL, CHASH_VAL],
-
- # headers doc - search by msgid.
- TYPE_MSGID_IDX: [KTYPE, MSGID_KEY],
-
- # content, headers doc
- TYPE_C_HASH_IDX: [KTYPE, CHASH_VAL],
-
- # attachment payload dedup
- TYPE_P_HASH_IDX: [KTYPE, PHASH_VAL],
-
- # messages
- TYPE_MBOX_SEEN_IDX: [KTYPE, MBOX_VAL, 'bool(seen)'],
- TYPE_MBOX_RECT_IDX: [KTYPE, MBOX_VAL, 'bool(recent)'],
- TYPE_MBOX_DEL_IDX: [KTYPE, MBOX_VAL, 'bool(deleted)'],
- TYPE_MBOX_RECT_SEEN_IDX: [KTYPE, MBOX_VAL,
- 'bool(recent)', 'bool(seen)'],
-
- # incoming queue
- JUST_MAIL_IDX: [INCOMING_KEY,
- "bool(%s)" % (ERROR_DECRYPTING_KEY,)],
-
- # the backward-compatible index, will be deprecated at 0.7
- JUST_MAIL_COMPAT_IDX: [INCOMING_KEY],
- }
-
- MBOX_KEY = MBOX_VAL
-
- EMPTY_MBOX = {
- TYPE_KEY: MBOX_KEY,
- TYPE_MBOX_VAL: MBoxParser.INBOX_NAME,
- SUBJECT_KEY: "",
- FLAGS_KEY: [],
- CLOSED_KEY: False,
- SUBSCRIBED_KEY: False,
- RW_KEY: 1,
- LAST_UID_KEY: 0
- }
fields = WithMsgFields # alias for convenience
diff --git a/src/leap/mail/imap/index.py b/src/leap/mail/imap/index.py
deleted file mode 100644
index ea35fff..0000000
--- a/src/leap/mail/imap/index.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# -*- coding: utf-8 -*-
-# index.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Index for SoledadBackedAccount, Mailbox and Messages.
-"""
-import logging
-
-from twisted.internet import defer
-
-from leap.common.check import leap_assert, leap_assert_type
-
-from leap.mail.imap.fields import fields
-
-
-logger = logging.getLogger(__name__)
-
-
-class IndexedDB(object):
- """
- Methods dealing with the index.
-
- This is a MixIn that needs access to the soledad instance,
- and also assumes that a INDEXES attribute is accessible to the instance.
-
- INDEXES must be a dictionary of type:
- {'index-name': ['field1', 'field2']}
- """
- # TODO we might want to move this to soledad itself, check
-
- _index_creation_deferreds = []
- index_ready = False
-
- def initialize_db(self):
- """
- Initialize the database.
- """
- leap_assert(self._soledad,
- "Need a soledad attribute accesible in the instance")
- leap_assert_type(self.INDEXES, dict)
- self._index_creation_deferreds = []
-
- def _on_indexes_created(ignored):
- self.index_ready = True
-
- def _create_index(name, expression):
- d = self._soledad.create_index(name, *expression)
- self._index_creation_deferreds.append(d)
-
- def _create_indexes(db_indexes):
- db_indexes = dict(db_indexes)
- for name, expression in fields.INDEXES.items():
- if name not in db_indexes:
- # The index does not yet exist.
- _create_index(name, expression)
- continue
-
- if expression == db_indexes[name]:
- # The index exists and is up to date.
- continue
- # The index exists but the definition is not what expected, so
- # we delete it and add the proper index expression.
- d1 = self._soledad.delete_index(name)
- d1.addCallback(lambda _: _create_index(name, expression))
-
- all_created = defer.gatherResults(self._index_creation_deferreds)
- all_created.addCallback(_on_indexes_created)
- return all_created
-
- # Ask the database for currently existing indexes.
- if not self._soledad:
- logger.debug("NO SOLEDAD ON IMAP INITIALIZATION")
- return
- if self._soledad is not None:
- d = self._soledad.list_indexes()
- d.addCallback(_create_indexes)
- return d
diff --git a/src/leap/mail/imap/interfaces.py b/src/leap/mail/imap/interfaces.py
index c906278..f8f25fa 100644
--- a/src/leap/mail/imap/interfaces.py
+++ b/src/leap/mail/imap/interfaces.py
@@ -20,6 +20,7 @@ Interfaces for the IMAP module.
from zope.interface import Interface, Attribute
+# TODO remove ----------------
class IMessageContainer(Interface):
"""
I am a container around the different documents that a message
@@ -38,6 +39,7 @@ class IMessageContainer(Interface):
"""
+# TODO remove --------------------
class IMessageStore(Interface):
"""
I represent a generic storage for LEAP Messages.
diff --git a/src/leap/mail/imap/mailbox.py b/src/leap/mail/imap/mailbox.py
index 3c1769a..ea54d33 100644
--- a/src/leap/mail/imap/mailbox.py
+++ b/src/leap/mail/imap/mailbox.py
@@ -1,6 +1,6 @@
# *- coding: utf-8 -*-
# mailbox.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013, 2014 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -18,6 +18,7 @@
Soledad Mailbox.
"""
import copy
+import re
import threading
import logging
import StringIO
@@ -27,6 +28,7 @@ import os
from collections import defaultdict
from twisted.internet import defer
+from twisted.internet import reactor
from twisted.internet.task import deferLater
from twisted.python import log
@@ -36,15 +38,18 @@ from zope.interface import implements
from leap.common import events as leap_events
from leap.common.events.events_pb2 import IMAP_UNREAD_MAIL
from leap.common.check import leap_assert, leap_assert_type
+from leap.mail.constants import INBOX_NAME
from leap.mail.decorators import deferred_to_thread
from leap.mail.utils import empty
from leap.mail.imap.fields import WithMsgFields, fields
from leap.mail.imap.messages import MessageCollection
from leap.mail.imap.messageparts import MessageWrapper
-from leap.mail.imap.parser import MBoxParser
logger = logging.getLogger(__name__)
+# TODO
+# [ ] Restore profile_cmd instrumentation
+
"""
If the environment variable `LEAP_SKIPNOTIFY` is set, we avoid
notifying clients of new messages. Use during stress tests.
@@ -71,7 +76,9 @@ if PROFILE_CMD:
d.addErrback(lambda f: log.msg(f.getTraceback()))
-class SoledadMailbox(WithMsgFields, MBoxParser):
+# TODO Rename to Mailbox
+# TODO Remove WithMsgFields
+class SoledadMailbox(WithMsgFields):
"""
A Soledad-backed IMAP mailbox.
@@ -115,7 +122,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
_last_uid_primed = {}
_known_uids_primed = {}
- def __init__(self, mbox, soledad, memstore, rw=1):
+ # TODO pass the collection to the constructor
+ # TODO pass the mbox_doc too
+ def __init__(self, mbox, store, rw=1):
"""
SoledadMailbox constructor. Needs to get passed a name, plus a
Soledad instance.
@@ -123,30 +132,21 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:param mbox: the mailbox name
:type mbox: str
- :param soledad: a Soledad instance.
- :type soledad: Soledad
-
- :param memstore: a MemoryStore instance
- :type memstore: MemoryStore
+ :param store:
+ :type store: Soledad
:param rw: read-and-write flag for this mailbox
:type rw: int
"""
leap_assert(mbox, "Need a mailbox name to initialize")
- leap_assert(soledad, "Need a soledad instance to initialize")
+ leap_assert(store, "Need a store instance to initialize")
- from twisted.internet import reactor
- self.reactor = reactor
-
- self.mbox = self._parse_mailbox_name(mbox)
+ self.mbox = normalize_mailbox(mbox)
self.rw = rw
- self._soledad = soledad
- self._memstore = memstore
-
- self.messages = MessageCollection(
- mbox=mbox, soledad=self._soledad, memstore=self._memstore)
+ self.store = store
+ self.messages = MessageCollection(mbox=mbox, soledad=store)
self._uidvalidity = None
# XXX careful with this get/set (it would be
@@ -214,7 +214,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
"""
return self._memstore.get_mbox_doc(self.mbox)
- # XXX the memstore->soledadstore method in memstore is not complete
def getFlags(self):
"""
Returns the flags defined for this mailbox.
@@ -227,7 +226,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
flags = self.INIT_FLAGS
return map(str, flags)
- # XXX the memstore->soledadstore method in memstore is not complete
def setFlags(self, flags):
"""
Sets flags for this mailbox.
@@ -468,8 +466,8 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
d = self._do_add_message(message, flags=flags, date=date,
notify_on_disk=notify_on_disk)
- if PROFILE_CMD:
- do_profile_cmd(d, "APPEND")
+ #if PROFILE_CMD:
+ #do_profile_cmd(d, "APPEND")
# XXX should review now that we're not using qtreactor.
# A better place for this would be the COPY/APPEND dispatcher
@@ -477,7 +475,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
# to work fine for now.
def notifyCallback(x):
- self.reactor.callLater(0, self.notify_new)
+ reactor.callLater(0, self.notify_new)
return x
d.addCallback(notifyCallback)
@@ -630,9 +628,9 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:rtype: deferred
"""
d = defer.Deferred()
- self.reactor.callInThread(self._do_fetch, messages_asked, uid, d)
- if PROFILE_CMD:
- do_profile_cmd(d, "FETCH")
+
+ # XXX do not need no thread...
+ reactor.callInThread(self._do_fetch, messages_asked, uid, d)
d.addCallback(self.cb_signal_unread_to_ui)
return d
@@ -800,7 +798,6 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
d.addCallback(self.__cb_signal_unread_to_ui)
return result
- @deferred_to_thread
def _get_unseen_deferred(self):
return self.getUnseenCount()
@@ -897,7 +894,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
:rtype: C{list} or C{Deferred}
"""
# TODO see if we can raise w/o interrupting flow
- #:raise IllegalQueryError: Raised when query is not valid.
+ # :raise IllegalQueryError: Raised when query is not valid.
# example query:
# ['UNDELETED', 'HEADER', 'Message-ID',
# '52D44F11.9060107@dev.bitmask.net']
@@ -991,7 +988,7 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
d.addCallback(createCopy)
d.addErrback(lambda f: log.msg(f.getTraceback()))
- @deferred_to_thread
+ #@deferred_to_thread
def _get_msg_copy(self, message):
"""
Get a copy of the fdoc for this message, and check whether
@@ -1049,3 +1046,22 @@ class SoledadMailbox(WithMsgFields, MBoxParser):
"""
return u"<SoledadMailbox: mbox '%s' (%s)>" % (
self.mbox, self.messages.count())
+
+
+def normalize_mailbox(name):
+ """
+ Return a normalized representation of the mailbox ``name``.
+
+ This method ensures that an eventual initial 'inbox' part of a
+ mailbox name is made uppercase.
+
+ :param name: the name of the mailbox
+ :type name: unicode
+
+ :rtype: unicode
+ """
+ _INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
+ if _INBOX_RE.match(name):
+ # ensure inital INBOX is uppercase
+ return INBOX_NAME + name[len(INBOX_NAME):]
+ return name
diff --git a/src/leap/mail/imap/messages.py b/src/leap/mail/imap/messages.py
index c761091..d47c8eb 100644
--- a/src/leap/mail/imap/messages.py
+++ b/src/leap/mail/imap/messages.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# messages.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2013, 2014 LEAP
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -19,30 +19,25 @@ LeapMessage and MessageCollection.
"""
import copy
import logging
-import re
import threading
import StringIO
from collections import defaultdict
-from email import message_from_string
from functools import partial
-from pycryptopp.hash import sha256
from twisted.mail import imap4
-from twisted.internet import defer, reactor
+from twisted.internet import reactor
from zope.interface import implements
from zope.proxy import sameProxiedObjects
from leap.common.check import leap_assert, leap_assert_type
from leap.common.decorators import memoized_method
from leap.common.mail import get_email_charset
-from leap.mail import walk
-from leap.mail.utils import first, find_charset, lowerdict, empty
-from leap.mail.utils import stringify_parts_map
-from leap.mail.decorators import deferred_to_thread
+from leap.mail.adaptors import soledad_indexes as indexes
+from leap.mail.constants import INBOX_NAME
+from leap.mail.utils import find_charset, empty
from leap.mail.imap.index import IndexedDB
from leap.mail.imap.fields import fields, WithMsgFields
-from leap.mail.imap.memorystore import MessageWrapper
from leap.mail.imap.messageparts import MessagePart, MessagePartDoc
from leap.mail.imap.parser import MBoxParser
@@ -59,9 +54,6 @@ logger = logging.getLogger(__name__)
# [ ] Delete incoming mail only after successful write!
# [ ] Remove UID from syncable db. Store only those indexes locally.
-MSGID_PATTERN = r"""<([\w@.]+)>"""
-MSGID_RE = re.compile(MSGID_PATTERN)
-
def try_unique_query(curried):
"""
@@ -90,28 +82,18 @@ def try_unique_query(curried):
logger.exception("Unhandled error %r" % exc)
-"""
-A dictionary that keeps one lock per mbox and uid.
-"""
-# XXX too much overhead?
-fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
+# FIXME remove-me
+#fdoc_locks = defaultdict(lambda: defaultdict(lambda: threading.Lock()))
-class LeapMessage(fields, MBoxParser):
+class IMAPMessage(fields, MBoxParser):
"""
The main representation of a message.
-
- It indexes the messages in one mailbox by a combination
- of uid+mailbox name.
"""
- # TODO this has to change.
- # Should index primarily by chash, and keep a local-only
- # UID table.
-
implements(imap4.IMessage)
- def __init__(self, soledad, uid, mbox, collection=None, container=None):
+ def __init__(self, soledad, uid, mbox):
"""
Initializes a LeapMessage.
@@ -129,76 +111,73 @@ class LeapMessage(fields, MBoxParser):
self._soledad = soledad
self._uid = int(uid) if uid is not None else None
self._mbox = self._parse_mailbox_name(mbox)
- self._collection = collection
- self._container = container
self.__chash = None
self.__bdoc = None
- # XXX make these properties public
-
- # XXX FIXME ------ the documents can be
- # deferreds too.... niice.
-
- @property
- def fdoc(self):
- """
- An accessor to the flags document.
- """
- if all(map(bool, (self._uid, self._mbox))):
- fdoc = None
- if self._container is not None:
- fdoc = self._container.fdoc
- if not fdoc:
- fdoc = self._get_flags_doc()
- if fdoc:
- fdoc_content = fdoc.content
- self.__chash = fdoc_content.get(
- fields.CONTENT_HASH_KEY, None)
- return fdoc
-
- @property
- def hdoc(self):
- """
- An accessor to the headers document.
- """
- container = self._container
- if container is not None:
- hdoc = self._container.hdoc
- if hdoc and not empty(hdoc.content):
- return hdoc
- hdoc = self._get_headers_doc()
-
- if container and not empty(hdoc.content):
+ # TODO collection and container are deprecated.
+
+ # TODO move to adaptor
+
+ #@property
+ #def fdoc(self):
+ #"""
+ #An accessor to the flags document.
+ #"""
+ #if all(map(bool, (self._uid, self._mbox))):
+ #fdoc = None
+ #if self._container is not None:
+ #fdoc = self._container.fdoc
+ #if not fdoc:
+ #fdoc = self._get_flags_doc()
+ #if fdoc:
+ #fdoc_content = fdoc.content
+ #self.__chash = fdoc_content.get(
+ #fields.CONTENT_HASH_KEY, None)
+ #return fdoc
+#
+ #@property
+ #def hdoc(self):
+ #"""
+ #An accessor to the headers document.
+ #"""
+ #container = self._container
+ #if container is not None:
+ #hdoc = self._container.hdoc
+ #if hdoc and not empty(hdoc.content):
+ #return hdoc
+ #hdoc = self._get_headers_doc()
+#
+ #if container and not empty(hdoc.content):
# mem-cache it
- hdoc_content = hdoc.content
- chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
- hdocs = {chash: hdoc_content}
- container.memstore.load_header_docs(hdocs)
- return hdoc
-
- @property
- def chash(self):
- """
- An accessor to the content hash for this message.
- """
- if not self.fdoc:
- return None
- if not self.__chash and self.fdoc:
- self.__chash = self.fdoc.content.get(
- fields.CONTENT_HASH_KEY, None)
- return self.__chash
-
- @property
- def bdoc(self):
- """
- An accessor to the body document.
- """
- if not self.hdoc:
- return None
- if not self.__bdoc:
- self.__bdoc = self._get_body_doc()
- return self.__bdoc
+ #hdoc_content = hdoc.content
+ #chash = hdoc_content.get(fields.CONTENT_HASH_KEY)
+ #hdocs = {chash: hdoc_content}
+ #container.memstore.load_header_docs(hdocs)
+ #return hdoc
+#
+ #@property
+ #def chash(self):
+ #"""
+ #An accessor to the content hash for this message.
+ #"""
+ #if not self.fdoc:
+ #return None
+ #if not self.__chash and self.fdoc:
+ #self.__chash = self.fdoc.content.get(
+ #fields.CONTENT_HASH_KEY, None)
+ #return self.__chash
+
+ #@property
+ #def bdoc(self):
+ #"""
+ #An accessor to the body document.
+ #"""
+ #if not self.hdoc:
+ #return None
+ #if not self.__bdoc:
+ #self.__bdoc = self._get_body_doc()
+ #return self.__bdoc
# IMessage implementation
@@ -209,8 +188,13 @@ class LeapMessage(fields, MBoxParser):
:return: uid for this message
:rtype: int
"""
+ # TODO ----> return lookup in local sqlcipher table.
return self._uid
+ # --------------------------------------------------------------
+ # TODO -- from here on, all the methods should be proxied to the
+ # instance of leap.mail.mail.Message
+
def getFlags(self):
"""
Retrieve the flags associated with this Message.
@@ -253,25 +237,24 @@ class LeapMessage(fields, MBoxParser):
REMOVE = -1
SET = 0
- with fdoc_locks[mbox][uid]:
- doc = self.fdoc
- if not doc:
- logger.warning(
- "Could not find FDOC for %r:%s while setting flags!" %
- (mbox, uid))
- return
- current = doc.content[self.FLAGS_KEY]
- if mode == APPEND:
- newflags = tuple(set(tuple(current) + flags))
- elif mode == REMOVE:
- newflags = tuple(set(current).difference(set(flags)))
- elif mode == SET:
- newflags = flags
- new_fdoc = {
- self.FLAGS_KEY: newflags,
- self.SEEN_KEY: self.SEEN_FLAG in newflags,
- self.DEL_KEY: self.DELETED_FLAG in newflags}
- self._collection.memstore.update_flags(mbox, uid, new_fdoc)
+ doc = self.fdoc
+ if not doc:
+ logger.warning(
+ "Could not find FDOC for %r:%s while setting flags!" %
+ (mbox, uid))
+ return
+ current = doc.content[self.FLAGS_KEY]
+ if mode == APPEND:
+ newflags = tuple(set(tuple(current) + flags))
+ elif mode == REMOVE:
+ newflags = tuple(set(current).difference(set(flags)))
+ elif mode == SET:
+ newflags = flags
+ new_fdoc = {
+ self.FLAGS_KEY: newflags,
+ self.SEEN_KEY: self.SEEN_FLAG in newflags,
+ self.DEL_KEY: self.DELETED_FLAG in newflags}
+ self._collection.memstore.update_flags(mbox, uid, new_fdoc)
return map(str, newflags)
@@ -371,9 +354,9 @@ class LeapMessage(fields, MBoxParser):
else:
logger.warning("No FLAGS doc for %s:%s" % (self._mbox,
self._uid))
- if not size:
+ #if not size:
# XXX fallback, should remove when all migrated.
- size = self.getBodyFile().len
+ #size = self.getBodyFile().len
return size
def getHeaders(self, negate, *names):
@@ -395,6 +378,9 @@ class LeapMessage(fields, MBoxParser):
# XXX refactor together with MessagePart method
headers = self._get_headers()
+
+ # XXX keep this in the imap imessage implementation,
+ # because the server impl. expects content-type to be present.
if not headers:
logger.warning("No headers found")
return {str('content-type'): str('')}
@@ -614,64 +600,23 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
(the u1db index) for all the headers documents for a given mailbox.
We use it to prefetch massively all the headers for a mailbox.
This is the second massive query, after fetching all the FLAGS, that
- a MUA will do in a case where we do not have local disk cache.
+ a typical IMAP MUA will do in a case where we do not have local disk cache.
"""
HDOCS_SET_DOC = "HDOCS_SET"
templates = {
- # Message Level
-
- FLAGS_DOC: {
- fields.TYPE_KEY: fields.TYPE_FLAGS_VAL,
- fields.UID_KEY: 1, # XXX moe to a local table
- fields.MBOX_KEY: fields.INBOX_VAL,
- fields.CONTENT_HASH_KEY: "",
-
- fields.SEEN_KEY: False,
- fields.DEL_KEY: False,
- fields.FLAGS_KEY: [],
- fields.MULTIPART_KEY: False,
- fields.SIZE_KEY: 0
- },
-
- HEADERS_DOC: {
- fields.TYPE_KEY: fields.TYPE_HEADERS_VAL,
- fields.CONTENT_HASH_KEY: "",
-
- fields.DATE_KEY: "",
- fields.SUBJECT_KEY: "",
-
- fields.HEADERS_KEY: {},
- fields.PARTS_MAP_KEY: {},
- },
-
- CONTENT_DOC: {
- fields.TYPE_KEY: fields.TYPE_CONTENT_VAL,
- fields.PAYLOAD_HASH_KEY: "",
- fields.LINKED_FROM_KEY: [],
- fields.CTYPE_KEY: "", # should index by this too
-
- # should only get inmutable headers parts
- # (for indexing)
- fields.HEADERS_KEY: {},
- fields.RAW_KEY: "",
- fields.PARTS_MAP_KEY: {},
- fields.HEADERS_KEY: {},
- fields.MULTIPART_KEY: False,
- },
-
# Mailbox Level
RECENT_DOC: {
- fields.TYPE_KEY: fields.TYPE_RECENT_VAL,
- fields.MBOX_KEY: fields.INBOX_VAL,
+ "type": indexes.RECENT,
+ "mbox": INBOX_NAME,
fields.RECENTFLAGS_KEY: [],
},
HDOCS_SET_DOC: {
- fields.TYPE_KEY: fields.TYPE_HDOCS_SET_VAL,
- fields.MBOX_KEY: fields.INBOX_VAL,
+ "type": indexes.HDOCS_SET,
+ "mbox": INBOX_NAME,
fields.HDOCS_SET_KEY: [],
}
@@ -681,8 +626,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
# Different locks for wrapping both the u1db document getting/setting
# and the property getting/settting in an atomic operation.
- # TODO we would abstract this to a SoledadProperty class
-
+ # TODO --- deprecate ! --- use SoledadDocumentWrapper + locks
_rdoc_lock = defaultdict(lambda: threading.Lock())
_rdoc_write_lock = defaultdict(lambda: threading.Lock())
_rdoc_read_lock = defaultdict(lambda: threading.Lock())
@@ -764,81 +708,9 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
rdoc[fields.MBOX_KEY] = self.mbox
self._soledad.create_doc(rdoc)
- @deferred_to_thread
- def _do_parse(self, raw):
- """
- Parse raw message and return it along with
- relevant information about its outer level.
-
- This is done in a separate thread, and the callback is passed
- to `_do_add_msg` method.
+ # --------------------------------------------------------------------
- :param raw: the raw message
- :type raw: StringIO or basestring
- :return: msg, parts, chash, size, multi
- :rtype: tuple
- """
- msg = message_from_string(raw)
- parts = walk.get_parts(msg)
- size = len(raw)
- chash = sha256.SHA256(raw).hexdigest()
- multi = msg.is_multipart()
- return msg, parts, chash, size, multi
-
- def _populate_flags(self, flags, uid, chash, size, multi):
- """
- Return a flags doc.
-
- XXX Missing DOC -----------
- """
- fd = self._get_empty_doc(self.FLAGS_DOC)
-
- fd[self.MBOX_KEY] = self.mbox
- fd[self.UID_KEY] = uid
- fd[self.CONTENT_HASH_KEY] = chash
- fd[self.SIZE_KEY] = size
- fd[self.MULTIPART_KEY] = multi
- if flags:
- fd[self.FLAGS_KEY] = flags
- fd[self.SEEN_KEY] = self.SEEN_FLAG in flags
- fd[self.DEL_KEY] = self.DELETED_FLAG in flags
- fd[self.RECENT_KEY] = True # set always by default
- return fd
-
- def _populate_headr(self, msg, chash, subject, date):
- """
- Return a headers doc.
-
- XXX Missing DOC -----------
- """
- headers = defaultdict(list)
- for k, v in msg.items():
- headers[k].append(v)
-
- # "fix" for repeated headers.
- for k, v in headers.items():
- newline = "\n%s: " % (k,)
- headers[k] = newline.join(v)
-
- lower_headers = lowerdict(headers)
- msgid = first(MSGID_RE.findall(
- lower_headers.get('message-id', '')))
-
- hd = self._get_empty_doc(self.HEADERS_DOC)
- hd[self.CONTENT_HASH_KEY] = chash
- hd[self.HEADERS_KEY] = headers
- hd[self.MSGID_KEY] = msgid
-
- if not subject and self.SUBJECT_FIELD in headers:
- hd[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
- else:
- hd[self.SUBJECT_KEY] = subject
-
- if not date and self.DATE_FIELD in headers:
- hd[self.DATE_KEY] = headers[self.DATE_FIELD]
- else:
- hd[self.DATE_KEY] = date
- return hd
+ # -----------------------------------------------------------------------
def _fdoc_already_exists(self, chash):
"""
@@ -885,86 +757,41 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
flags = tuple()
leap_assert_type(flags, tuple)
- # TODO return soledad deferred instead
- observer = defer.Deferred()
- d = self._do_parse(raw)
- d.addCallback(lambda result: reactor.callInThread(
- self._do_add_msg, result, flags, subject, date,
- notify_on_disk, observer))
- return observer
+ # TODO ---- proxy to MessageCollection addMessage
+
+ #observer = defer.Deferred()
+ #d = self._do_parse(raw)
+ #d.addCallback(lambda result: reactor.callInThread(
+ #self._do_add_msg, result, flags, subject, date,
+ #notify_on_disk, observer))
+ #return observer
+
+ # TODO ---------------------------------------------------
+ # move this to leap.mail.adaptors.soledad
- # Called in thread
def _do_add_msg(self, parse_result, flags, subject,
date, notify_on_disk, observer):
"""
- Helper that creates a new message document.
- Here lives the magic of the leap mail. Well, in soledad, really.
-
- See `add_msg` docstring for parameter info.
-
- :param parse_result: a tuple with the results of `self._do_parse`
- :type parse_result: tuple
- :param observer: a deferred that will be fired with the message
- uid when the adding succeed.
- :type observer: deferred
"""
- # TODO signal that we can delete the original message!-----
- # when all the processing is done.
-
- # TODO add the linked-from info !
- # TODO add reference to the original message
-
msg, parts, chash, size, multi = parse_result
+ # XXX move to SoledadAdaptor write operation ... ???
# check for uniqueness --------------------------------
# Watch out! We're reserving a UID right after this!
existing_uid = self._fdoc_already_exists(chash)
if existing_uid:
msg = self.get_msg_by_uid(existing_uid)
-
- # We can say the observer that we're done
- # TODO return soledad deferred instead
reactor.callFromThread(observer.callback, existing_uid)
msg.setFlags((fields.DELETED_FLAG,), -1)
return
+ # TODO move UID autoincrement to MessageCollection.addMessage(mailbox)
# TODO S2 -- get FUCKING UID from autoincremental table
- uid = self.memstore.increment_last_soledad_uid(self.mbox)
-
- # We can say the observer that we're done at this point, but
- # before that we should make sure it has no serious consequences
- # if we're issued, for instance, a fetch command right after...
- # reactor.callFromThread(observer.callback, uid)
- # if we did the notify, we need to invalidate the deferred
- # so not to try to fire it twice.
- # observer = None
-
- fd = self._populate_flags(flags, uid, chash, size, multi)
- hd = self._populate_headr(msg, chash, subject, date)
-
- body_phash_fun = [walk.get_body_phash_simple,
- walk.get_body_phash_multi][int(multi)]
- body_phash = body_phash_fun(walk.get_payloads(msg))
- parts_map = walk.walk_msg_tree(parts, body_phash=body_phash)
-
- # add parts map to header doc
- # (body, multi, part_map)
- for key in parts_map:
- hd[key] = parts_map[key]
- del parts_map
+ #uid = self.memstore.increment_last_soledad_uid(self.mbox)
+ #self.set_recent_flag(uid)
- hd = stringify_parts_map(hd)
- # The MessageContainer expects a dict, one-indexed
- cdocs = dict(enumerate(walk.get_raw_docs(msg, parts), 1))
-
- self.set_recent_flag(uid)
- msg_container = MessageWrapper(fd, hd, cdocs)
-
- # TODO S1 -- just pass this to memstore and return that deferred.
- self.memstore.create_message(
- self.mbox, uid, msg_container,
- observer=observer, notify_on_disk=notify_on_disk)
+ # ------------------------------------------------------------
#
# getters: specific queries
@@ -1073,6 +900,10 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
the query failed.
:rtype: SoledadDocument or None.
"""
+ # USED from:
+ # [ ] duplicated fdoc detection
+ # [ ] _get_uid_from_msgidCb
+
# FIXME ----- use deferreds.
curried = partial(
self._soledad.get_from_index,
@@ -1205,51 +1036,52 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
if msg_container is not None:
if mem_only:
- msg = LeapMessage(None, uid, self.mbox, collection=self,
+ msg = IMAPMessage(None, uid, self.mbox, collection=self,
container=msg_container)
else:
# We pass a reference to soledad just to be able to retrieve
# missing parts that cannot be found in the container, like
# the content docs after a copy.
- msg = LeapMessage(self._soledad, uid, self.mbox,
+ msg = IMAPMessage(self._soledad, uid, self.mbox,
collection=self, container=msg_container)
else:
- msg = LeapMessage(self._soledad, uid, self.mbox, collection=self)
+ msg = IMAPMessage(self._soledad, uid, self.mbox, collection=self)
if not msg.does_exist():
return None
return msg
- def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
- """
- Get all documents for the selected mailbox of the
- passed type. By default, it returns the flag docs.
-
- If you want acess to the content, use __iter__ instead
-
- :return: a Deferred, that will fire with a list of u1db documents
- :rtype: Deferred (promise of list of SoledadDocument)
- """
- if _type not in fields.__dict__.values():
- raise TypeError("Wrong type passed to get_all_docs")
-
+ # FIXME --- used where ? ---------------------------------------------
+ #def get_all_docs(self, _type=fields.TYPE_FLAGS_VAL):
+ #"""
+ #Get all documents for the selected mailbox of the
+ #passed type. By default, it returns the flag docs.
+#
+ #If you want acess to the content, use __iter__ instead
+#
+ #:return: a Deferred, that will fire with a list of u1db documents
+ #:rtype: Deferred (promise of list of SoledadDocument)
+ #"""
+ #if _type not in fields.__dict__.values():
+ #raise TypeError("Wrong type passed to get_all_docs")
+#
# FIXME ----- either raise or return a deferred wrapper.
- if sameProxiedObjects(self._soledad, None):
- logger.warning('Tried to get messages but soledad is None!')
- return []
-
- def get_sorted_docs(docs):
- all_docs = [doc for doc in docs]
+ #if sameProxiedObjects(self._soledad, None):
+ #logger.warning('Tried to get messages but soledad is None!')
+ #return []
+#
+ #def get_sorted_docs(docs):
+ #all_docs = [doc for doc in docs]
# inneficient, but first let's grok it and then
# let's worry about efficiency.
# XXX FIXINDEX -- should implement order by in soledad
# FIXME ----------------------------------------------
- return sorted(all_docs, key=lambda item: item.content['uid'])
-
- d = self._soledad.get_from_index(
- fields.TYPE_MBOX_IDX, _type, self.mbox)
- d.addCallback(get_sorted_docs)
- return d
+ #return sorted(all_docs, key=lambda item: item.content['uid'])
+#
+ #d = self._soledad.get_from_index(
+ #fields.TYPE_MBOX_IDX, _type, self.mbox)
+ #d.addCallback(get_sorted_docs)
+ #return d
def all_soledad_uid_iter(self):
"""
@@ -1350,7 +1182,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:returns: a list of LeapMessages
:rtype: list
"""
- return [LeapMessage(self._soledad, docid, self.mbox, collection=self)
+ return [IMAPMessage(self._soledad, docid, self.mbox, collection=self)
for docid in self.unseen_iter()]
# recent messages
@@ -1384,7 +1216,7 @@ class MessageCollection(WithMsgFields, IndexedDB, MBoxParser):
:returns: iterator of dicts with content for all messages.
:rtype: iterable
"""
- return (LeapMessage(self._soledad, docuid, self.mbox, collection=self)
+ return (IMAPMessage(self._soledad, docuid, self.mbox, collection=self)
for docuid in self.all_uid_iter())
def __repr__(self):
diff --git a/src/leap/mail/imap/parser.py b/src/leap/mail/imap/parser.py
deleted file mode 100644
index 4a801b0..0000000
--- a/src/leap/mail/imap/parser.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-# parser.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-"""
-Mail parser mixin.
-"""
-import re
-
-
-class MBoxParser(object):
- """
- Utility function to parse mailbox names.
- """
- INBOX_NAME = "INBOX"
- INBOX_RE = re.compile(INBOX_NAME, re.IGNORECASE)
-
- def _parse_mailbox_name(self, name):
- """
- Return a normalized representation of the mailbox C{name}.
-
- This method ensures that an eventual initial 'inbox' part of a
- mailbox name is made uppercase.
-
- :param name: the name of the mailbox
- :type name: unicode
-
- :rtype: unicode
- """
- if self.INBOX_RE.match(name):
- # ensure inital INBOX is uppercase
- return self.INBOX_NAME + name[len(self.INBOX_NAME):]
- return name
diff --git a/src/leap/mail/imap/tests/test_imap.py b/src/leap/mail/imap/tests/test_imap.py
index dd4294c..5af499f 100644
--- a/src/leap/mail/imap/tests/test_imap.py
+++ b/src/leap/mail/imap/tests/test_imap.py
@@ -94,6 +94,8 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
MessageCollection interface in this particular TestCase
"""
super(MessageCollectionTestCase, self).setUp()
+
+ # TODO deprecate memstore
memstore = MemoryStore()
self.messages = MessageCollection("testmbox%s" % (self.count,),
self._soledad, memstore=memstore)
diff --git a/src/leap/mail/imap/tests/utils.py b/src/leap/mail/imap/tests/utils.py
index 9a3868c..920eeb0 100644
--- a/src/leap/mail/imap/tests/utils.py
+++ b/src/leap/mail/imap/tests/utils.py
@@ -51,6 +51,7 @@ class SimpleClient(imap4.IMAP4Client):
self.transport.loseConnection()
+# XXX move to common helper
def initialize_soledad(email, gnupg_home, tempdir):
"""
Initializes soledad by hand
@@ -110,9 +111,7 @@ class IMAP4HelperMixin(BaseLeapTest):
"""
Setup method for each test.
- Initializes and run a LEAP IMAP4 Server,
- but passing the same Soledad instance (it's costly to initialize),
- so we have to be sure to restore state across tests.
+ Initializes and run a LEAP IMAP4 Server.
"""
self.old_path = os.environ['PATH']
self.old_home = os.environ['HOME']
@@ -172,19 +171,17 @@ class IMAP4HelperMixin(BaseLeapTest):
def tearDown(self):
"""
tearDown method called after each test.
-
- Deletes all documents in the Index, and deletes
- instances of server and client.
"""
try:
self._soledad.close()
+ except Exception:
+ print "ERROR WHILE CLOSING SOLEDAD"
+ finally:
os.environ["PATH"] = self.old_path
os.environ["HOME"] = self.old_home
# safety check
assert 'leap_tests-' in self.tempdir
shutil.rmtree(self.tempdir)
- except Exception:
- print "ERROR WHILE CLOSING SOLEDAD"
def populateMessages(self):
"""
@@ -223,5 +220,3 @@ class IMAP4HelperMixin(BaseLeapTest):
def loopback(self):
return loopback.loopbackAsync(self.server, self.client)
-
-