Adapt IMAP to latest Soledad api.

* Also fix some tests that where not up-to-date with code.

4 files changed, 155 insertions, 92 deletions

diff --git a/mail/src/leap/mail/imap/fetch.py b/mail/src/leap/mail/imap/fetch.py
index df5d046..48a45e6 100644
--- a/mail/src/leap/mail/imap/fetch.py
+++ b/mail/src/leap/mail/imap/fetch.py
@@ -104,8 +104,8 @@ class LeapIncomingMail(object):
         """
         Process a successfully decrypted message.
 
-        :param doc: a LeapDocument instance containing the incoming message
-        :type doc: LeapDocument
+        :param doc: a SoledadDocument instance containing the incoming message
+        :type doc: SoledadDocument
 
         :param data: the json-encoded, decrypted content of the incoming
                      message
diff --git a/mail/src/leap/mail/imap/server.py b/mail/src/leap/mail/imap/server.py
index 2b66ba6..813d850 100644
--- a/mail/src/leap/mail/imap/server.py
+++ b/mail/src/leap/mail/imap/server.py
@@ -37,7 +37,7 @@ from twisted.python import log
 
 from leap.common.check import leap_assert, leap_assert_type
 from leap.soledad import Soledad
-from leap.soledad.backends.sqlcipher import SQLCipherDatabase
+from leap.soledad.sqlcipher import SQLCipherDatabase
 
 logger = logging.getLogger(__name__)
 
@@ -81,7 +81,7 @@ class WithMsgFields(object):
 
     INBOX_VAL = "inbox"
 
-    # Flags for LeapDocument for indexing.
+    # Flags for SoledadDocument for indexing.
     SEEN_KEY = "seen"
     RECENT_KEY = "recent"
 
@@ -233,7 +233,7 @@ class SoledadBackedAccount(WithMsgFields, IndexedDB):
         :param name: the name of the mailbox
         :type name: str
 
-        :rtype: LeapDocument
+        :rtype: SoledadDocument
         """
         name = name.upper()
         doc = self._soledad.get_from_index(
@@ -580,9 +580,9 @@ class LeapMessage(WithMsgFields):
         """
         Initializes a LeapMessage.
 
-        :param doc: A LeapDocument containing the internal
+        :param doc: A SoledadDocument containing the internal
                     representation of the message
-        :type doc: LeapDocument
+        :type doc: SoledadDocument
         """
         self._doc = doc
 
@@ -620,13 +620,13 @@ class LeapMessage(WithMsgFields):
         """
         Sets the flags for this message
 
-        Returns a LeapDocument that needs to be updated by the caller.
+        Returns a SoledadDocument that needs to be updated by the caller.
 
         :param flags: the flags to update in the message.
         :type flags: sequence of str
 
-        :return: a LeapDocument instance
-        :rtype: LeapDocument
+        :return: a SoledadDocument instance
+        :rtype: SoledadDocument
         """
         log.msg('setting flags')
         doc = self._doc
@@ -639,13 +639,13 @@ class LeapMessage(WithMsgFields):
         """
         Adds flags to this message.
 
-        Returns a LeapDocument that needs to be updated by the caller.
+        Returns a SoledadDocument that needs to be updated by the caller.
 
         :param flags: the flags to add to the message.
         :type flags: sequence of str
 
-        :return: a LeapDocument instance
-        :rtype: LeapDocument
+        :return: a SoledadDocument instance
+        :rtype: SoledadDocument
         """
         oldflags = self.getFlags()
         return self.setFlags(list(set(flags + oldflags)))
@@ -654,13 +654,13 @@ class LeapMessage(WithMsgFields):
         """
         Remove flags from this message.
 
-        Returns a LeapDocument that needs to be updated by the caller.
+        Returns a SoledadDocument that needs to be updated by the caller.
 
         :param flags: the flags to be removed from the message.
         :type flags: sequence of str
 
-        :return: a LeapDocument instance
-        :rtype: LeapDocument
+        :return: a SoledadDocument instance
+        :rtype: SoledadDocument
         """
         oldflags = self.getFlags()
         return self.setFlags(list(set(oldflags) - set(flags)))
@@ -751,6 +751,7 @@ class LeapMessage(WithMsgFields):
         :rtype: dict
         """
         headers = self._get_headers()
+        names = map(lambda s: s.upper(), names)
         if negate:
             cond = lambda key: key.upper() not in names
         else:
@@ -771,6 +772,21 @@ class LeapMessage(WithMsgFields):
     def getSubPart(part):
         return None
 
+    #
+    # accessors
+    #
+
+    def __getitem__(self, key):
+        """
+        Return the content of the message document.
+
+        @param key: The key
+        @type key: str
+
+        @return: The content value indexed by C{key} or None
+        @rtype: str
+        """
+        return self._doc.content.get(key, None)
 
 class MessageCollection(WithMsgFields, IndexedDB):
     """
@@ -828,7 +844,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
 
         self.mbox = mbox.upper()
         self._soledad = soledad
-        #self.db = db
+        self.initialize_db()
         self._parser = Parser()
 
     def _get_empty_msg(self):
@@ -887,11 +903,18 @@ class MessageCollection(WithMsgFields, IndexedDB):
 
         # XXX get lower case for keys?
         content[self.HEADERS_KEY] = headers
-        content[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
+        # set subject based on message headers and eventually replace by
+        # subject given as param
+        if self.SUBJECT_FIELD in headers:
+            content[self.SUBJECT_KEY] = headers[self.SUBJECT_FIELD]
+        if subject is not None:
+            content[self.SUBJECT_KEY] = subject
         content[self.RAW_KEY] = stringify(raw)
 
-        if not date:
+        if not date and self.DATE_FIELD in headers:
             content[self.DATE_KEY] = headers[self.DATE_FIELD]
+        else:
+            content[self.DATE_KEY] = date
 
         # ...should get a sanity check here.
         content[self.UID_KEY] = uid
@@ -903,7 +926,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         Removes a message.
 
         :param msg: a u1db doc containing the message
-        :type msg: LeapDocument
+        :type msg: SoledadDocument
         """
         self._soledad.delete_doc(msg)
 
@@ -916,9 +939,9 @@ class MessageCollection(WithMsgFields, IndexedDB):
         :param uid: the message uid to query by
         :type uid: int
 
-        :return: A LeapDocument instance matching the query,
+        :return: A SoledadDocument instance matching the query,
                  or None if not found.
-        :rtype: LeapDocument
+        :rtype: SoledadDocument
         """
         docs = self._soledad.get_from_index(
             SoledadBackedAccount.TYPE_MBOX_UID_IDX,
@@ -946,7 +969,7 @@ class MessageCollection(WithMsgFields, IndexedDB):
         If you want acess to the content, use __iter__ instead
 
         :return: a list of u1db documents
-        :rtype: list of LeapDocument
+        :rtype: list of SoledadDocument
         """
         # XXX this should return LeapMessage instances
         return self._soledad.get_from_index(
@@ -1117,8 +1140,8 @@ class SoledadMailbox(WithMsgFields):
         """
         Returns mailbox document.
 
-        :return: A LeapDocument containing this mailbox.
-        :rtype: LeapDocument
+        :return: A SoledadDocument containing this mailbox.
+        :rtype: SoledadDocument
         """
         query = self._soledad.get_from_index(
             SoledadBackedAccount.TYPE_MBOX_IDX,
@@ -1133,15 +1156,15 @@ class SoledadMailbox(WithMsgFields):
         :returns: tuple of flags for this mailbox
         :rtype: tuple of str
         """
-        return map(str, self.INIT_FLAGS)
+        #return map(str, self.INIT_FLAGS)
 
         # XXX CHECK against thunderbird XXX
 
-        #mbox = self._get_mbox()
-        #if not mbox:
-            #return None
-        #flags = mbox.content.get(self.FLAGS_KEY, [])
-        #return map(str, flags)
+        mbox = self._get_mbox()
+        if not mbox:
+            return None
+        flags = mbox.content.get(self.FLAGS_KEY, [])
+        return map(str, flags)
 
     def setFlags(self, flags):
         """
@@ -1442,7 +1465,7 @@ class SoledadMailbox(WithMsgFields):
         """
         docs = self.messages.get_all()
         for doc in docs:
-            self.messages.db.delete_doc(doc)
+            self.messages._soledad.delete_doc(doc)
 
     def _update(self, doc):
         """
diff --git a/mail/src/leap/mail/imap/tests/__init__.py b/mail/src/leap/mail/imap/tests/__init__.py
index 315d649..fdeda76 100644
--- a/mail/src/leap/mail/imap/tests/__init__.py
+++ b/mail/src/leap/mail/imap/tests/__init__.py
@@ -17,13 +17,13 @@ def run():
     """xxx fill me in"""
     pass
 
+import os
 import u1db
 
 from leap.common.testing.basetest import BaseLeapTest
 
 from leap.soledad import Soledad
-from leap.soledad.util import GPGWrapper
-from leap.soledad.backends.leap_backend import LeapDocument
+from leap.soledad.document import SoledadDocument
 
 
 #-----------------------------------------------------------------------------
@@ -38,30 +38,65 @@ class BaseSoledadIMAPTest(BaseLeapTest):
     """
 
     def setUp(self):
-        # config info
-        self.gnupg_home = "%s/gnupg" % self.tempdir
-        self.db1_file = "%s/db1.u1db" % self.tempdir
-        self.db2_file = "%s/db2.u1db" % self.tempdir
-        self.email = 'leap@leap.se'
         # open test dbs
+        self.db1_file = os.path.join(
+            self.tempdir, "db1.u1db")
+        self.db2_file = os.path.join(
+            self.tempdir, "db2.u1db")
+
         self._db1 = u1db.open(self.db1_file, create=True,
-                              document_factory=LeapDocument)
+                              document_factory=SoledadDocument)
         self._db2 = u1db.open(self.db2_file, create=True,
-                              document_factory=LeapDocument)
-
-        # initialize soledad by hand so we can control keys
-        self._soledad = Soledad(self.email, gnupg_home=self.gnupg_home,
-                                bootstrap=False,
-                                prefix=self.tempdir)
-        self._soledad._init_dirs()
-        self._soledad._gpg = GPGWrapper(gnupghome=self.gnupg_home)
-
-        if not self._soledad._has_privkey():
-            self._soledad._set_privkey(PRIVATE_KEY)
-        if not self._soledad._has_symkey():
-            self._soledad._gen_symkey()
-        self._soledad._load_symkey()
-        self._soledad._init_db()
+                              document_factory=SoledadDocument)
+
+        # soledad config info
+        self.email = 'leap@leap.se'
+        secrets_path = os.path.join(
+            self.tempdir, Soledad.STORAGE_SECRETS_FILE_NAME)
+        local_db_path = os.path.join(
+            self.tempdir, Soledad.LOCAL_DATABASE_FILE_NAME)
+        server_url = ''
+        cert_file = None
+
+        self._soledad = self._soledad_instance(
+            self.email, '123',
+            secrets_path=secrets_path,
+            local_db_path=local_db_path,
+            server_url=server_url,
+            cert_file=cert_file)
+
+    def _soledad_instance(self, uuid, passphrase, secrets_path, local_db_path,
+                          server_url, cert_file):
+        """
+        Return a Soledad instance for tests.
+        """
+        # mock key fetching and storing so Soledad doesn't fail when trying to
+        # reach the server.
+        Soledad._fetch_keys_from_shared_db = Mock(return_value=None)
+        Soledad._assert_keys_in_shared_db = Mock(return_value=None)
+
+        # instantiate soledad
+        def _put_doc_side_effect(doc):
+            self._doc_put = doc
+
+        class MockSharedDB(object):
+
+            get_doc = Mock(return_value=None)
+            put_doc = Mock(side_effect=_put_doc_side_effect)
+
+            def __call__(self):
+                return self
+
+        Soledad._shared_db = MockSharedDB()
+
+        return Soledad(
+            uuid,
+            passphrase,
+            secrets_path=secrets_path,
+            local_db_path=local_db_path,
+            server_url=server_url,
+            cert_file=cert_file,
+        )
 
     def tearDown(self):
         self._db1.close()
diff --git a/mail/src/leap/mail/imap/tests/test_imap.py b/mail/src/leap/mail/imap/tests/test_imap.py
index 6b6c24e..8804fe0 100644
--- a/mail/src/leap/mail/imap/tests/test_imap.py
+++ b/mail/src/leap/mail/imap/tests/test_imap.py
@@ -31,24 +31,18 @@ try:
 except ImportError:
     from StringIO import StringIO
 
-#import codecs
-#import locale
 import os
 import types
 import tempfile
 import shutil
 
 
-#from zope.interface import implements
+from mock import Mock
+
 
-#from twisted.mail.imap4 import MessageSet
 from twisted.mail import imap4
 from twisted.protocols import loopback
 from twisted.internet import defer
-#from twisted.internet import error
-#from twisted.internet import reactor
-#from twisted.internet import interfaces
-#from twisted.internet.task import Clock
 from twisted.trial import unittest
 from twisted.python import util, log
 from twisted.python import failure
@@ -59,8 +53,6 @@ import twisted.cred.checkers
 import twisted.cred.credentials
 import twisted.cred.portal
 
-#from twisted.test.proto_helpers import StringTransport, StringTransportWithDisconnection
-
 
 #import u1db
 
@@ -68,8 +60,6 @@ from leap.common.testing.basetest import BaseLeapTest
 from leap.mail.imap.server import SoledadMailbox
 from leap.mail.imap.server import SoledadBackedAccount
 from leap.mail.imap.server import MessageCollection
-#from leap.mail.imap.tests import PUBLIC_KEY
-#from leap.mail.imap.tests import PRIVATE_KEY
 
 from leap.soledad import Soledad
 from leap.soledad import SoledadCrypto
@@ -107,19 +97,23 @@ def initialize_soledad(email, gnupg_home, tempdir):
     server_url = "http://provider"
     cert_file = ""
 
+    class MockSharedDB(object):
+
+        get_doc = Mock(return_value=None)
+        put_doc = Mock()
+
+        def __call__(self):
+            return self
+
+    Soledad._shared_db = MockSharedDB()
+
     _soledad = Soledad(
-        uuid,  # user's uuid, obtained through signal events
-        passphrase,  # how to get this?
-        secret_path,  # how to get this?
-        local_db_path,  # how to get this?
-        server_url,  # can be None for now
-        cert_file,
-        bootstrap=False)
-    _soledad._init_dirs()
-    _soledad._crypto = SoledadCrypto(_soledad)
-    _soledad._shared_db = None
-    _soledad._init_keys()
-    _soledad._init_db()
+        uuid,
+        passphrase,
+        secret_path,
+        local_db_path,
+        server_url,
+        cert_file)
 
     return _soledad
 
@@ -253,9 +247,9 @@ class IMAP4HelperMixin(BaseLeapTest):
         #cls.db2_file = "%s/db2.u1db" % cls.tempdir
         # open test dbs
         #cls._db1 = u1db.open(cls.db1_file, create=True,
-                              #document_factory=LeapDocument)
+                              #document_factory=SoledadDocument)
         #cls._db2 = u1db.open(cls.db2_file, create=True,
-                              #document_factory=LeapDocument)
+                              #document_factory=SoledadDocument)
 
         # initialize soledad by hand so we can control keys
         cls._soledad = initialize_soledad(
@@ -401,12 +395,21 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Test empty message and collection
         """
-        em = self.messages.get_empty_msg()
-        self.assertEqual(em,
-                         {"subject": "", "seen": False,
-                          "flags": [], "mailbox": "inbox",
-                          "mbox-uid": 1,
-                          "raw": ""})
+        em = self.messages._get_empty_msg()
+        self.assertEqual(
+            em,
+            {
+                "date": '',
+                "flags": [],
+                "headers": {},
+                "mbox": "inbox",
+                "raw": "",
+                "recent": True,
+                "seen": False,
+                "subject": "",
+                "type": "msg",
+                "uid": 1,
+            })
         self.assertEqual(self.messages.count(), 0)
 
     def testFilterByMailbox(self):
@@ -419,13 +422,13 @@ class MessageCollectionTestCase(IMAP4HelperMixin, unittest.TestCase):
         mc.add_msg('', subject="test3")
         self.assertEqual(self.messages.count(), 3)
 
-        newmsg = mc.get_empty_msg()
+        newmsg = mc._get_empty_msg()
         newmsg['mailbox'] = "mailbox/foo"
         newmsg['subject'] = "test another mailbox"
-        mc.db.create_doc(newmsg)
+        mc._soledad.create_doc(newmsg)
         self.assertEqual(mc.count(), 3)
-        self.assertEqual(len(mc.db.get_from_index(mc.MAILBOX_INDEX, "*")),
-                         4)
+        self.assertEqual(
+            len(mc._soledad.get_from_index(mc.TYPE_IDX, "*")), 4)
 
 
 class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
@@ -561,10 +564,13 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         Try deleting a mailbox with sub-folders, and \NoSelect flag set.
         An exception is expected
+
+        Obs: this test will fail if SoledadMailbox returns hardcoded flags.
         """
         SimpleLEAPServer.theAccount.addMailbox('delete')
         to_delete = SimpleLEAPServer.theAccount.getMailbox('delete')
         to_delete.setFlags((r'\Noselect',))
+        to_delete.getFlags()
         SimpleLEAPServer.theAccount.addMailbox('delete/me')
 
         def login():
@@ -1180,7 +1186,6 @@ class LeapIMAP4ServerTestCase(IMAP4HelperMixin, unittest.TestCase):
         """
         name = 'mailbox-close'
         self.server.theAccount.addMailbox(name)
-        #import ipdb; ipdb.set_trace()
 
         m = SimpleLEAPServer.theAccount.getMailbox(name)
         m.messages.add_msg('', subject="Message 1",