Functional SoledadBackedAccount and LeapMailboxes

The imap service is launched from the tac file,
and still needs some information to be provided in separate
config files that stub much of the initialization parameters.

working fetch and store methods.
tested with offlineimap and thunderbird.
several mailboxes might be broken.

4 files changed, 436 insertions, 0 deletions

diff --git a/src/leap/mail/imap/service/README.rst b/src/leap/mail/imap/service/README.rst
new file mode 100644
index 0000000..2cca9b3
--- /dev/null
+++ b/src/leap/mail/imap/service/README.rst
@@ -0,0 +1,39 @@
+testing the service
+===================
+
+Run the twisted service::
+
+        twistd -n -y imap-server.tac
+
+And use offlineimap for tests::
+
+        offlineimap -c LEAPofflineimapRC-tests
+
+minimal offlineimap configuration
+---------------------------------
+
+[general]
+accounts = leap-local
+
+[Account leap-local]
+localrepository = LocalLeap
+remoterepository = RemoteLeap
+
+[Repository LocalLeap]
+type = Maildir
+localfolders = ~/LEAPMail/Mail
+
+[Repository RemoteLeap]
+type = IMAP
+ssl = no
+remotehost = localhost
+remoteport = 9930
+remoteuser = user
+remotepass = pass
+
+debugging
+---------
+
+Use ngrep to obtain logs of the sequences::
+
+        sudo ngrep -d lo -W byline port 9930
diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac
new file mode 100644
index 0000000..e491e06
--- /dev/null
+++ b/src/leap/mail/imap/service/imap-server.tac
@@ -0,0 +1,230 @@
+import ConfigParser
+import datetime
+import os
+from functools import partial
+
+from xdg import BaseDirectory
+
+from twisted.application import internet, service
+from twisted.internet.protocol import ServerFactory
+from twisted.mail import imap4
+from twisted.python import log
+
+from leap.common.check import leap_assert
+from leap.mail.imap.server import SoledadBackedAccount
+from leap.mail.imap.fetch import LeapIncomingMail
+from leap.soledad import Soledad
+#from leap.soledad import SoledadCrypto
+
+# Some constants
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# The port in which imap service will run
+IMAP_PORT = 9930
+
+# The period between succesive checks of the incoming mail
+# queue (in seconds)
+INCOMING_CHECK_PERIOD = 10
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+
+class LeapIMAPServer(imap4.IMAP4Server):
+    """
+    An IMAP4 Server with mailboxes backed by soledad
+    """
+    def __init__(self, *args, **kwargs):
+        # pop extraneous arguments
+        soledad = kwargs.pop('soledad', None)
+        user = kwargs.pop('user', None)
+        gpg = kwargs.pop('gpg', None)
+        leap_assert(soledad, "need a soledad instance")
+        leap_assert(user, "need a user in the initialization")
+
+        # initialize imap server!
+        imap4.IMAP4Server.__init__(self, *args, **kwargs)
+
+        # we should initialize the account here,
+        # but we move it to the factory so we can
+        # populate the test account properly (and only once
+        # per session)
+
+        # theAccount = SoledadBackedAccount(
+        #     user, soledad=soledad)
+
+        # ---------------------------------
+        # XXX pre-populate acct for tests!!
+        # populate_test_account(theAccount)
+        # ---------------------------------
+        #self.theAccount = theAccount
+
+    def lineReceived(self, line):
+        log.msg('rcv: %s' % line)
+        imap4.IMAP4Server.lineReceived(self, line)
+
+    def authenticateLogin(self, username, password):
+        # all is allowed so far. use realm instead
+        return imap4.IAccount, self.theAccount, lambda: None
+
+
+class IMAPAuthRealm(object):
+    """
+    dummy authentication realm
+    """
+    theAccount = None
+
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        return imap4.IAccount, self.theAccount, lambda: None
+
+
+class LeapIMAPFactory(ServerFactory):
+    """
+    Factory for a IMAP4 server with soledad remote sync and gpg-decryption
+    capabilities.
+    """
+
+    def __init__(self, user, soledad, gpg=None):
+        self._user = user
+        self._soledad = soledad
+        self._gpg = gpg
+
+        theAccount = SoledadBackedAccount(
+            user, soledad=soledad)
+
+        # ---------------------------------
+        # XXX pre-populate acct for tests!!
+        # populate_test_account(theAccount)
+        # ---------------------------------
+        self.theAccount = theAccount
+
+    def buildProtocol(self, addr):
+        "Return a protocol suitable for the job."
+        imapProtocol = LeapIMAPServer(
+            user=self._user,
+            soledad=self._soledad,
+            gpg=self._gpg)
+        imapProtocol.theAccount = self.theAccount
+        imapProtocol.factory = self
+        return imapProtocol
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# Let's rock...
+#
+# XXX initialize gpg
+
+#from leap.mail.imap.tests import PUBLIC_KEY
+#from leap.mail.imap.tests import PRIVATE_KEY
+#from leap.soledad.util import GPGWrapper
+
+
+def initialize_mailbox_soledad(user_uuid, soledad_pass, server_url,
+                       	       server_pemfile, token):
+    """
+    Initializes soledad by hand
+
+    :param user_uuid:
+    :param soledad_pass:
+    :param server_url:
+    :param server_pemfile:
+    :param token:
+
+    :rtype: Soledad instance
+    """
+    #XXX do we need a separate instance for the mailbox db?
+
+    base_config = BaseDirectory.xdg_config_home
+    secret_path = os.path.join(
+        base_config, "leap", "soledad", "%s.secret" % user_uuid)
+    soledad_path = os.path.join(
+        base_config, "leap", "soledad", "%s-mailbox.db" % user_uuid)
+
+
+    _soledad = Soledad(
+        user_uuid,
+	soledad_pass,
+        secret_path,
+        soledad_path,
+	server_url,
+        server_pemfile,
+        token,
+        bootstrap=True)
+    #_soledad._init_dirs()
+    #_soledad._crypto = SoledadCrypto(_soledad)
+    #_soledad._shared_db = None
+    #_soledad._init_keys()
+    #_soledad._init_db()
+
+    return _soledad
+
+'''
+mail_sample = open('rfc822.message').read()
+def populate_test_account(acct):
+    """
+    Populates inbox for testing purposes
+    """
+    print "populating test account!"
+    inbox = acct.getMailbox('inbox')
+    inbox.addMessage(mail_sample, ("\\Foo", "\\Recent",), date="Right now2")
+'''
+
+def incoming_check(fetcher):
+    """
+    Check incoming queue. To be called periodically.
+    """
+    #log.msg("checking incoming queue...")
+    fetcher.fetch()
+
+
+#######################################################################
+# XXX STUBBED! We need to get this in the instantiation from the client
+
+config = ConfigParser.ConfigParser()
+config.read([os.path.expanduser('~/.config/leap/mail/mail.conf')])
+
+userID = config.get('mail', 'address')
+privkey = open(os.path.expanduser('~/.config/leap/mail/privkey')).read()
+nickserver_url = ""
+
+d = {}
+
+for key in ('uid', 'passphrase', 'server', 'pemfile', 'token'):
+    d[key] = config.get('mail', key)
+
+soledad = initialize_mailbox_soledad(
+    d['uid'],
+    d['passphrase'],
+    d['server'],
+    d['pemfile'],
+    d['token'])
+gpg = None
+
+# import the private key ---- should sync it from remote!
+from leap.common.keymanager.openpgp import OpenPGPScheme
+opgp = OpenPGPScheme(soledad)
+opgp.put_ascii_key(privkey)
+
+from leap.common.keymanager import KeyManager
+keym = KeyManager(userID, nickserver_url, soledad, d['token'])
+
+#import ipdb; ipdb.set_trace()
+
+
+factory = LeapIMAPFactory(userID, soledad, gpg)
+
+application = service.Application("LEAP IMAP4 Local Service")
+imapService = internet.TCPServer(IMAP_PORT, factory)
+imapService.setServiceParent(application)
+
+fetcher = LeapIncomingMail(
+    keym,
+    d['uid'],
+    d['passphrase'],
+    d['server'],
+    d['pemfile'],
+    d['token'],
+    factory.theAccount)
+
+
+incoming_check_for_acct = partial(incoming_check, fetcher)
+internet.TimerService(
+    INCOMING_CHECK_PERIOD,
+    incoming_check_for_acct).setServiceParent(application)
diff --git a/src/leap/mail/imap/service/notes.txt b/src/leap/mail/imap/service/notes.txt
new file mode 100644
index 0000000..623e122
--- /dev/null
+++ b/src/leap/mail/imap/service/notes.txt
@@ -0,0 +1,81 @@
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* OK [CAPABILITY IMAP4rev1 IDLE NAMESPACE] Twisted IMAP4rev1 Ready.
+
+##
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ1 CAPABILITY.
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* CAPABILITY IMAP4rev1 IDLE NAMESPACE.
+NCLJ1 OK CAPABILITY completed.
+
+##
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ2 LOGIN user "pass".
+
+#
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+NCLJ2 OK LOGIN succeeded.
+
+##
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ3 CAPABILITY.
+
+#
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* CAPABILITY IMAP4rev1 IDLE NAMESPACE.
+NCLJ3 OK CAPABILITY completed.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ4 LIST "" "".
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX".
+NCLJ4 OK LIST completed.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ5 LIST "" "*".
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* LIST (\Seen \Answered \Flagged \Deleted \Draft \Recent List) "/" "INBOX".
+NCLJ5 OK LIST completed.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ6 SELECT INBOX.
+
+#
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* 0 EXISTS.
+* 3 RECENT.
+* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List).
+* OK [UIDVALIDITY 42].
+NCLJ6 OK [READ-WRITE] SELECT successful.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ7 EXAMINE INBOX.
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* 0 EXISTS.
+* 3 RECENT.
+* FLAGS (\Seen \Answered \Flagged \Deleted \Draft \Recent List).
+* OK [UIDVALIDITY 42].
+NCLJ7 OK [READ-ONLY] EXAMINE successful.
+
+#
+T 127.0.0.1:42866 -> 127.0.0.1:9930 [AP]
+NCLJ8 LOGOUT.
+
+##
+T 127.0.0.1:9930 -> 127.0.0.1:42866 [AP]
+* BYE Nice talking to you.
+NCLJ8 OK LOGOUT successful.
+
+
diff --git a/src/leap/mail/imap/service/rfc822.message b/src/leap/mail/imap/service/rfc822.message
new file mode 100644
index 0000000..ee97ab9
--- /dev/null
+++ b/src/leap/mail/imap/service/rfc822.message
@@ -0,0 +1,86 @@
+Return-Path: <twisted-commits-admin@twistedmatrix.com>
+Delivered-To: exarkun@meson.dyndns.org
+Received: from localhost [127.0.0.1]
+	by localhost with POP3 (fetchmail-6.2.1)
+	for exarkun@localhost (single-drop); Thu, 20 Mar 2003 14:50:20 -0500 (EST)
+Received: from pyramid.twistedmatrix.com (adsl-64-123-27-105.dsl.austtx.swbell.net [64.123.27.105])
+	by intarweb.us (Postfix) with ESMTP id 4A4A513EA4
+	for <exarkun@meson.dyndns.org>; Thu, 20 Mar 2003 14:49:27 -0500 (EST)
+Received: from localhost ([127.0.0.1] helo=pyramid.twistedmatrix.com)
+	by pyramid.twistedmatrix.com with esmtp (Exim 3.35 #1 (Debian))
+	id 18w648-0007Vl-00; Thu, 20 Mar 2003 13:51:04 -0600
+Received: from acapnotic by pyramid.twistedmatrix.com with local (Exim 3.35 #1 (Debian))
+	id 18w63j-0007VK-00
+	for <twisted-commits@twistedmatrix.com>; Thu, 20 Mar 2003 13:50:39 -0600
+To: twisted-commits@twistedmatrix.com
+From: etrepum CVS <etrepum@twistedmatrix.com>
+Reply-To: twisted-python@twistedmatrix.com
+X-Mailer: CVSToys
+Message-Id: <E18w63j-0007VK-00@pyramid.twistedmatrix.com>
+Subject: [Twisted-commits] rebuild now works on python versions from 2.2.0 and up.
+Sender: twisted-commits-admin@twistedmatrix.com
+Errors-To: twisted-commits-admin@twistedmatrix.com
+X-BeenThere: twisted-commits@twistedmatrix.com
+X-Mailman-Version: 2.0.11
+Precedence: bulk
+List-Help: <mailto:twisted-commits-request@twistedmatrix.com?subject=help>
+List-Post: <mailto:twisted-commits@twistedmatrix.com>
+List-Subscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+	<mailto:twisted-commits-request@twistedmatrix.com?subject=subscribe>
+List-Id: <twisted-commits.twistedmatrix.com>
+List-Unsubscribe: <http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits>,
+	<mailto:twisted-commits-request@twistedmatrix.com?subject=unsubscribe>
+List-Archive: <http://twistedmatrix.com/pipermail/twisted-commits/>
+Date: Thu, 20 Mar 2003 13:50:39 -0600
+
+Modified files:
+Twisted/twisted/python/rebuild.py 1.19 1.20
+
+Log message:
+rebuild now works on python versions from 2.2.0 and up.
+
+
+ViewCVS links:
+http://twistedmatrix.com/users/jh.twistd/viewcvs/cgi/viewcvs.cgi/twisted/python/rebuild.py.diff?r1=text&tr1=1.19&r2=text&tr2=1.20&cvsroot=Twisted
+
+Index: Twisted/twisted/python/rebuild.py
+diff -u Twisted/twisted/python/rebuild.py:1.19 Twisted/twisted/python/rebuild.py:1.20
+--- Twisted/twisted/python/rebuild.py:1.19	Fri Jan 17 13:50:49 2003
++++ Twisted/twisted/python/rebuild.py	Thu Mar 20 11:50:08 2003
+@@ -206,15 +206,27 @@
+             clazz.__dict__.clear()
+             clazz.__getattr__ = __getattr__
+             clazz.__module__ = module.__name__
++    if newclasses:
++        import gc
++        if (2, 2, 0) <= sys.version_info[:3] < (2, 2, 2):
++            hasBrokenRebuild = 1
++            gc_objects = gc.get_objects()
++        else:
++            hasBrokenRebuild = 0
+     for nclass in newclasses:
+         ga = getattr(module, nclass.__name__)
+         if ga is nclass:
+             log.msg("WARNING: new-class %s not replaced by reload!" % reflect.qual(nclass))
+         else:
+-            import gc
+-            for r in gc.get_referrers(nclass):
+-                if isinstance(r, nclass):
++            if hasBrokenRebuild:
++                for r in gc_objects:
++                    if not getattr(r, '__class__', None) is nclass:
++                        continue
+                     r.__class__ = ga
++            else:
++                for r in gc.get_referrers(nclass):
++                    if getattr(r, '__class__', None) is nclass:
++                        r.__class__ = ga
+     if doLog:
+         log.msg('')
+         log.msg('  (fixing   %s): ' % str(module.__name__))
+
+
+_______________________________________________
+Twisted-commits mailing list
+Twisted-commits@twistedmatrix.com
+http://twistedmatrix.com/cgi-bin/mailman/listinfo/twisted-commits