From 62b0cd6301b7097dfa2776b677ab3c7d27f60d7b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Dec 2013 14:10:14 -0400 Subject: Split the near-2k loc file into more handy modules. ...aaaand not a single fuck was given that day! --- src/leap/mail/imap/service/imap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 8756ddc..26e14c3 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -32,7 +32,7 @@ logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager -from leap.mail.imap.server import SoledadBackedAccount +from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail from leap.soledad.client import Soledad -- cgit v1.2.3 From 4ba5d5b405e3c6a6bc997df2073ffc8ea3fa75a9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 7 Jan 2014 11:34:08 -0400 Subject: Second stage of the new year's storage rewrite. * documents of only three types: * flags * headers * content * add algorithm for walking the parsed message tree. * treat special cases like a multipart with a single part. * modify add_msg to use the walk routine * modify twisted interfaces to use the new storage schema. * tests for different multipart cases * fix multipart detection typo in the fetch This is a merge proposal for the 0.5.0-rc3. known bugs ---------- Some things are still know not to work well at this point (some cases of multipart messages do not display the bodies). IMAP server also is left in a bad internal state after a logout/login. --- src/leap/mail/imap/service/imap.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 26e14c3..234996d 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -87,6 +87,8 @@ class LeapIMAPServer(imap4.IMAP4Server): :param line: the line from the server, without the line delimiter. :type line: str """ + print "RECV: STATE (%s)" % self.state + if "login" in line.lower(): # avoid to log the pass, even though we are using a dummy auth # by now. -- cgit v1.2.3 From f9a9a695526afea8eef4ccba8904fb215acc43dd Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 8 Jan 2014 23:29:18 -0400 Subject: add a flag to be able to close the session --- src/leap/mail/imap/service/imap.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 234996d..dfd4862 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -71,15 +71,6 @@ class LeapIMAPServer(imap4.IMAP4Server): # populate the test account properly (and only once # per session) - # theAccount = SoledadBackedAccount( - # user, soledad=soledad) - - # --------------------------------- - # XXX pre-populate acct for tests!! - # populate_test_account(theAccount) - # --------------------------------- - #self.theAccount = theAccount - def lineReceived(self, line): """ Attempt to parse a single line from the server. @@ -88,6 +79,9 @@ class LeapIMAPServer(imap4.IMAP4Server): :type line: str """ print "RECV: STATE (%s)" % self.state + if self.theAccount.closed is True and self.state != "unauth": + log.msg("Closing the session. State: unauth") + self.state = "unauth" if "login" in line.lower(): # avoid to log the pass, even though we are using a dummy auth -- cgit v1.2.3 From 241cde270f1ef37aa33332934869a58b143885d1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sat, 11 Jan 2014 20:31:08 -0400 Subject: add offline flag --- src/leap/mail/imap/service/imap.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index dfd4862..c48e5c5 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -178,6 +178,7 @@ def run_service(*args, **kwargs): check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD) userid = kwargs.get('userid', None) leap_check(userid is not None, "need an user id") + offline = kwargs.get('offline', False) uuid = soledad._get_uuid() factory = LeapIMAPFactory(uuid, userid, soledad) @@ -187,12 +188,15 @@ def run_service(*args, **kwargs): try: tport = reactor.listenTCP(port, factory, interface="localhost") - fetcher = LeapIncomingMail( - keymanager, - soledad, - factory.theAccount, - check_period, - userid) + if not offline: + fetcher = LeapIncomingMail( + keymanager, + soledad, + factory.theAccount, + check_period, + userid) + else: + fetcher = None except CannotListenError: logger.error("IMAP Service failed to start: " "cannot listen in port %s" % (port,)) @@ -200,7 +204,7 @@ def run_service(*args, **kwargs): logger.error("Error launching IMAP service: %r" % (exc,)) else: # all good. - fetcher.start_loop() + # (the caller has still to call fetcher.start_loop) logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) return fetcher, tport, factory -- cgit v1.2.3 From 90f4338da088394ade1663871a23b8fb0a4c0d66 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Jan 2014 17:05:24 -0400 Subject: Performance improvement on FLAGS-only FETCH * Compute the intersection set of the uids on a FETCH, so we avoid iterating through the non-existant UIDs. * Dispatch FLAGS query to our specialized method, that fetches all the flags documents and return objects that only specify one subset of the MessagePart interface, apt to render flags quickly with less queries overhead. * Overwrite the do_FETCH command in the imap Server to use fetch_flags. * Use deferLater for a better dispatch of tasks in the reactor. --- src/leap/mail/imap/service/imap.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index c48e5c5..e877869 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -22,6 +22,7 @@ from copy import copy import logging from twisted.internet.protocol import ServerFactory +from twisted.internet.defer import maybeDeferred from twisted.internet.error import CannotListenError from twisted.mail import imap4 from twisted.python import log @@ -78,7 +79,6 @@ class LeapIMAPServer(imap4.IMAP4Server): :param line: the line from the server, without the line delimiter. :type line: str """ - print "RECV: STATE (%s)" % self.state if self.theAccount.closed is True and self.state != "unauth": log.msg("Closing the session. State: unauth") self.state = "unauth" @@ -89,7 +89,7 @@ class LeapIMAPServer(imap4.IMAP4Server): msg = line[:7] + " [...]" else: msg = copy(line) - log.msg('rcv: %s' % msg) + log.msg('rcv (%s): %s' % (self.state, msg)) imap4.IMAP4Server.lineReceived(self, line) def authenticateLogin(self, username, password): @@ -111,6 +111,39 @@ class LeapIMAPServer(imap4.IMAP4Server): leap_events.signal(IMAP_CLIENT_LOGIN, "1") return imap4.IAccount, self.theAccount, lambda: None + def do_FETCH(self, tag, messages, query, uid=0): + """ + Overwritten fetch dispatcher to use the fast fetch_flags + method + """ + log.msg("LEAP Overwritten fetch...") + if not query: + self.sendPositiveResponse(tag, 'FETCH complete') + return # XXX ??? + + cbFetch = self._IMAP4Server__cbFetch + ebFetch = self._IMAP4Server__ebFetch + + if str(query[0]) == "flags": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_flags, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + else: + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) + + select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, + imap4.IMAP4Server.arg_fetchatt) + class IMAPAuthRealm(object): """ -- cgit v1.2.3 From 1069e7b9470fb63f70a43fa72407fcd4276d550d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Jan 2014 16:43:18 -0400 Subject: Update service initialization file This will need to place a configuration file with: * userid * uuid * password (optional) Use it for even faster startup times, and running under the native twisted reactor. --- src/leap/mail/imap/service/imap-server.tac | 182 +++++++++++++++++++++-------- 1 file changed, 135 insertions(+), 47 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index da72cae..b65bb17 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -1,69 +1,157 @@ +# -*- coding: utf-8 -*- +# imap-server.tac +# Copyright (C) 2013,2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +TAC file for initialization of the imap service using twistd. + +Use this for debugging and testing the imap server using a native reactor. + +For now, and for debugging/testing purposes, you need +to pass a config file with the following structure: + +[leap_mail] +userid = "user@provider" +uuid = "deadbeefdeadabad" +passwd = "supersecret" # optional, will get prompted if not found. +""" import ConfigParser +import getpass import os +import sys -from leap.soledad.client import Soledad +from leap.keymanager import KeyManager from leap.mail.imap.service import imap -from leap.common.config import get_path_prefix - - -config = ConfigParser.ConfigParser() -config.read([os.path.expanduser('~/.config/leap/mail/mail.conf')]) - -userID = config.get('mail', 'address') -privkey = open(os.path.expanduser('~/.config/leap/mail/privkey')).read() -nickserver_url = "" +from leap.soledad.client import Soledad -d = {} +from twisted.application import service, internet -for key in ('uid', 'passphrase', 'server', 'pemfile', 'token'): - d[key] = config.get('mail', key) +# TODO should get this initializers from some authoritative mocked source +# We might want to put them the soledad itself. -def initialize_soledad_mailbox(user_uuid, soledad_pass, server_url, - server_pemfile, token): +def initialize_soledad(uuid, email, passwd, + secrets, localdb, + gnupg_home, tempdir): """ Initializes soledad by hand - :param user_uuid: - :param soledad_pass: - :param server_url: - :param server_pemfile: - :param token: - + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir :rtype: Soledad instance """ + # XXX TODO unify with an authoritative source of mocks + # for soledad (or partial initializations). + # This is copied from the imap tests. + + server_url = "http://provider" + cert_file = "" - base_config = get_path_prefix() + class Mock(object): + def __init__(self, return_value=None): + self._return = return_value - secret_path = os.path.join( - base_config, "leap", "soledad", "%s.secret" % user_uuid) - soledad_path = os.path.join( - base_config, "leap", "soledad", "%s-mailbox.db" % user_uuid) + def __call__(self, *args, **kwargs): + return self._return - _soledad = Soledad( - user_uuid, - soledad_pass, - secret_path, - soledad_path, + class MockSharedDB(object): + + get_doc = Mock() + put_doc = Mock() + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) + + def __call__(self): + return self + + Soledad._shared_db = MockSharedDB() + soledad = Soledad( + uuid, + passwd, + secrets, + localdb, server_url, - server_pemfile, - token) + cert_file) + + return soledad + +###################################################################### +# Remember to set your config files, see module documentation above! +###################################################################### + +print "[+] Running LEAP IMAP Service" + + +bmconf = os.environ.get("LEAP_MAIL_CONF", "") +if not bmconf: + print "[-] Please set LEAP_MAIL_CONF environment variable pointing to your config." + sys.exit(1) +SECTION = "leap_mail" +cp = ConfigParser.ConfigParser() +cp.read(bmconf) + +userid = cp.get(SECTION, "userid") +uuid = cp.get(SECTION, "uuid") +passwd = unicode(cp.get(SECTION, "passwd")) + +# XXX get this right from the environment variable !!! +port = 1984 + +if not userid or not uuid: + print "[-] Config file missing userid or uuid field" + sys.exit(1) + +if not passwd: + passwd = unicode(getpass.getpass("Soledad passphrase: ")) + + +secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,)) +localdb = os.path.expanduser("~/.config/leap/soledad/%s.db" % (uuid,)) + +# XXX Is this really used? Should point it to user var dirs defined in xdg? +gnupg_home = "/tmp/" +tempdir = "/tmp/" + +################################################### + +# Ad-hoc soledad/keymanager initialization. + +soledad = initialize_soledad(uuid, userid, passwd, secrets, localdb, gnupg_home, tempdir) +km_args = (userid, "https://localhost", soledad) +km_kwargs = { + "session_id": "", + "ca_cert_path": "", + "api_uri": "", + "api_version": "", + "uid": uuid, + "gpgbinary": "/usr/bin/gpg" +} +keymanager = KeyManager(*km_args, **km_kwargs) + +################################################## - return _soledad +# Ok, let's expose the application object for the twistd application +# framework to pick up from here... -soledad = initialize_soledad_mailbox( - d['uid'], - d['passphrase'], - d['server'], - d['pemfile'], - d['token']) -# import the private key ---- should sync it from remote! -from leap.common.keymanager.openpgp import OpenPGPScheme -opgp = OpenPGPScheme(soledad) -opgp.put_ascii_key(privkey) +def getIMAPService(): + factory = imap.LeapIMAPFactory(uuid, userid, soledad) + return internet.TCPServer(port, factory, interface="localhost") -from leap.common.keymanager import KeyManager -keymanager = KeyManager(userID, nickserver_url, soledad, d['token']) -imap.run_service(soledad, keymanager) +application = service.Application("LEAP IMAP Application") +service = getIMAPService() +service.setServiceParent(application) -- cgit v1.2.3 From 77def3fb8d8b8106ebb341e6cb7ee9987ce6e2e9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Jan 2014 17:13:05 -0400 Subject: Dispatch the flags query if it's the only one. ie, we got something like FETCH 1:* (FLAGS) but not for FETCH 1:* (FLAGS INTERNALDATE) --- src/leap/mail/imap/service/imap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index e877869..8c5b488 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -124,7 +124,9 @@ class LeapIMAPServer(imap4.IMAP4Server): cbFetch = self._IMAP4Server__cbFetch ebFetch = self._IMAP4Server__ebFetch - if str(query[0]) == "flags": + print "QUERY: ", query + + if len(query) == 1 and str(query[0]) == "flags": self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator maybeDeferred( -- cgit v1.2.3 From 61454c82de35778d27bcd1a2c89fe20ee3d5b142 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Jan 2014 17:15:27 -0400 Subject: patch UIDVALIDITY response for conformance to the spec testimap was choking on this. --- src/leap/mail/imap/service/imap.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 8c5b488..6e03456 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -146,6 +146,36 @@ class LeapIMAPServer(imap4.IMAP4Server): select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, imap4.IMAP4Server.arg_fetchatt) + def _cbSelectWork(self, mbox, cmdName, tag): + """ + Callback for selectWork, patched to avoid conformance errors due to + incomplete UIDVALIDITY line. + """ + if mbox is None: + self.sendNegativeResponse(tag, 'No such mailbox') + return + if '\\noselect' in [s.lower() for s in mbox.getFlags()]: + self.sendNegativeResponse(tag, 'Mailbox cannot be selected') + return + + flags = mbox.getFlags() + self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') + self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') + self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) + + # Patched ------------------------------------------------------- + # imaptest was complaining about the incomplete line, we're adding + # "UIDs valid" here. + self.sendPositiveResponse( + None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity()) + # ---------------------------------------------------------------- + + s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY' + mbox.addListener(self) + self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName)) + self.state = 'select' + self.mbox = mbox + class IMAPAuthRealm(object): """ -- cgit v1.2.3 From 9f9701d42be385aa9a6d7e72fd10104b0025971b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Jan 2014 22:01:20 -0400 Subject: Separate RECENT Flag to a mailbox document. this way we avoid a bunch of writes. --- src/leap/mail/imap/service/imap.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 6e03456..a3ef098 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -24,6 +24,7 @@ import logging from twisted.internet.protocol import ServerFactory from twisted.internet.defer import maybeDeferred from twisted.internet.error import CannotListenError +from twisted.internet.task import deferLater from twisted.mail import imap4 from twisted.python import log from twisted import cred @@ -116,6 +117,7 @@ class LeapIMAPServer(imap4.IMAP4Server): Overwritten fetch dispatcher to use the fast fetch_flags method """ + from twisted.internet import reactor log.msg("LEAP Overwritten fetch...") if not query: self.sendPositiveResponse(tag, 'FETCH complete') @@ -124,8 +126,6 @@ class LeapIMAPServer(imap4.IMAP4Server): cbFetch = self._IMAP4Server__cbFetch ebFetch = self._IMAP4Server__ebFetch - print "QUERY: ", query - if len(query) == 1 and str(query[0]) == "flags": self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator @@ -141,11 +141,32 @@ class LeapIMAPServer(imap4.IMAP4Server): self.mbox.fetch, messages, uid=uid ).addCallback( cbFetch, tag, query, uid - ).addErrback(ebFetch, tag) + ).addErrback( + ebFetch, tag) + + deferLater(reactor, + 2, self.mbox.unset_recent_flags, messages) + deferLater(reactor, 1, self.mbox.signal_unread_to_ui) select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, imap4.IMAP4Server.arg_fetchatt) + def do_COPY(self, tag, messages, mailbox, uid=0): + from twisted.internet import reactor + imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid) + deferLater(reactor, + 2, self.mbox.unset_recent_flags, messages) + deferLater(reactor, 1, self.mbox.signal_unread_to_ui) + + select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset, + imap4.IMAP4Server.arg_astring) + + def notifyNew(self, ignored): + """ + Notify new messages to listeners. + """ + self.mbox.notify_new() + def _cbSelectWork(self, mbox, cmdName, tag): """ Callback for selectWork, patched to avoid conformance errors due to @@ -177,6 +198,7 @@ class LeapIMAPServer(imap4.IMAP4Server): self.mbox = mbox + class IMAPAuthRealm(object): """ Dummy authentication realm. Do not use in production! -- cgit v1.2.3 From 9ef1cd79397d811575826025b924c615e6ce2aa4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 17 Jan 2014 02:51:31 -0400 Subject: Add a fetch_headers for mass-header fetch queries --- src/leap/mail/imap/service/imap.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index a3ef098..a1d3ab7 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -123,6 +123,9 @@ class LeapIMAPServer(imap4.IMAP4Server): self.sendPositiveResponse(tag, 'FETCH complete') return # XXX ??? + print "QUERY ", query + print query[0] + cbFetch = self._IMAP4Server__cbFetch ebFetch = self._IMAP4Server__ebFetch @@ -134,6 +137,14 @@ class LeapIMAPServer(imap4.IMAP4Server): ).addCallback( cbFetch, tag, query, uid ).addErrback(ebFetch, tag) + elif len(query) == 1 and str(query[0]) == "rfc822.header": + self._oldTimeout = self.setTimeout(None) + # no need to call iter, we get a generator + maybeDeferred( + self.mbox.fetch_headers, messages, uid=uid + ).addCallback( + cbFetch, tag, query, uid + ).addErrback(ebFetch, tag) else: self._oldTimeout = self.setTimeout(None) # no need to call iter, we get a generator @@ -198,7 +209,6 @@ class LeapIMAPServer(imap4.IMAP4Server): self.mbox = mbox - class IMAPAuthRealm(object): """ Dummy authentication realm. Do not use in production! -- cgit v1.2.3 From 22c106a7306446a3fa9689f5942a86a53ec884b4 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 21 Jan 2014 01:04:37 -0400 Subject: workaround for recursionlimit due to qtreactor --- src/leap/mail/imap/service/imap.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index a1d3ab7..ad22da6 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -49,6 +49,25 @@ from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN +###################################################### +# Temporary workaround for RecursionLimit when using +# qt4reactor. Do remove when we move to poll or select +# reactor, which do not show those problems. See #4974 +import resource +import sys + +try: + sys.setrecursionlimit(10**6) +except Exception: + print "Error setting recursion limit" +try: + # Increase max stack size from 8MB to 256MB + resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1)) +except Exception: + print "Error setting stack size" + +###################################################### + class LeapIMAPServer(imap4.IMAP4Server): """ @@ -118,14 +137,10 @@ class LeapIMAPServer(imap4.IMAP4Server): method """ from twisted.internet import reactor - log.msg("LEAP Overwritten fetch...") if not query: self.sendPositiveResponse(tag, 'FETCH complete') return # XXX ??? - print "QUERY ", query - print query[0] - cbFetch = self._IMAP4Server__cbFetch ebFetch = self._IMAP4Server__ebFetch -- cgit v1.2.3 From 4ae6ad57a0f80143e3ded867c1fdd2264804a775 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 21 Jan 2014 19:22:09 -0400 Subject: memory store for append/fetch/copy --- src/leap/mail/imap/service/imap.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index ad22da6..71b9950 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -36,6 +36,7 @@ from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail +from leap.mail.imap.memorystore import MemoryStore from leap.soledad.client import Soledad # The default port in which imap service will run @@ -69,6 +70,8 @@ except Exception: ###################################################### +# TODO move this to imap.server + class LeapIMAPServer(imap4.IMAP4Server): """ An IMAP4 Server with mailboxes backed by soledad @@ -256,11 +259,15 @@ class LeapIMAPFactory(ServerFactory): self._uuid = uuid self._userid = userid self._soledad = soledad + self._memstore = MemoryStore() theAccount = SoledadBackedAccount( - uuid, soledad=soledad) + uuid, soledad=soledad, + memstore=self._memstore) self.theAccount = theAccount + # XXX how to pass the store along? + def buildProtocol(self, addr): "Return a protocol suitable for the job." imapProtocol = LeapIMAPServer( @@ -323,3 +330,14 @@ def run_service(*args, **kwargs): # not ok, signal error. leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port)) + + def checkpoint(self): + """ + Called when the client issues a CHECK command. + + This should perform any checkpoint operations required by the server. + It may be a long running operation, but may not block. If it returns + a deferred, the client will only be informed of success (or failure) + when the deferred's callback (or errback) is invoked. + """ + return None -- cgit v1.2.3 From 23b2a2c10ddd6dbb15e3f532d526ac0a53bd788b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 23 Jan 2014 00:27:19 -0400 Subject: move server to its own file --- src/leap/mail/imap/service/imap.py | 180 +------------------------------------ 1 file changed, 3 insertions(+), 177 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 71b9950..3f99da6 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -17,17 +17,11 @@ """ Imap service initialization """ -from copy import copy - import logging from twisted.internet.protocol import ServerFactory -from twisted.internet.defer import maybeDeferred from twisted.internet.error import CannotListenError -from twisted.internet.task import deferLater from twisted.mail import imap4 -from twisted.python import log -from twisted import cred logger = logging.getLogger(__name__) @@ -37,6 +31,7 @@ from leap.keymanager import KeyManager from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail from leap.mail.imap.memorystore import MemoryStore +from leap.mail.imap.server import LeapIMAPServer from leap.soledad.client import Soledad # The default port in which imap service will run @@ -48,7 +43,6 @@ INCOMING_CHECK_PERIOD = 60 from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START -from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN ###################################################### # Temporary workaround for RecursionLimit when using @@ -70,163 +64,6 @@ except Exception: ###################################################### -# TODO move this to imap.server - -class LeapIMAPServer(imap4.IMAP4Server): - """ - An IMAP4 Server with mailboxes backed by soledad - """ - def __init__(self, *args, **kwargs): - # pop extraneous arguments - soledad = kwargs.pop('soledad', None) - uuid = kwargs.pop('uuid', None) - userid = kwargs.pop('userid', None) - leap_assert(soledad, "need a soledad instance") - leap_assert_type(soledad, Soledad) - leap_assert(uuid, "need a user in the initialization") - - self._userid = userid - - # initialize imap server! - imap4.IMAP4Server.__init__(self, *args, **kwargs) - - # we should initialize the account here, - # but we move it to the factory so we can - # populate the test account properly (and only once - # per session) - - def lineReceived(self, line): - """ - Attempt to parse a single line from the server. - - :param line: the line from the server, without the line delimiter. - :type line: str - """ - if self.theAccount.closed is True and self.state != "unauth": - log.msg("Closing the session. State: unauth") - self.state = "unauth" - - if "login" in line.lower(): - # avoid to log the pass, even though we are using a dummy auth - # by now. - msg = line[:7] + " [...]" - else: - msg = copy(line) - log.msg('rcv (%s): %s' % (self.state, msg)) - imap4.IMAP4Server.lineReceived(self, line) - - def authenticateLogin(self, username, password): - """ - Lookup the account with the given parameters, and deny - the improper combinations. - - :param username: the username that is attempting authentication. - :type username: str - :param password: the password to authenticate with. - :type password: str - """ - # XXX this should use portal: - # return portal.login(cred.credentials.UsernamePassword(user, pass) - if username != self._userid: - # bad username, reject. - raise cred.error.UnauthorizedLogin() - # any dummy password is allowed so far. use realm instead! - leap_events.signal(IMAP_CLIENT_LOGIN, "1") - return imap4.IAccount, self.theAccount, lambda: None - - def do_FETCH(self, tag, messages, query, uid=0): - """ - Overwritten fetch dispatcher to use the fast fetch_flags - method - """ - from twisted.internet import reactor - if not query: - self.sendPositiveResponse(tag, 'FETCH complete') - return # XXX ??? - - cbFetch = self._IMAP4Server__cbFetch - ebFetch = self._IMAP4Server__ebFetch - - if len(query) == 1 and str(query[0]) == "flags": - self._oldTimeout = self.setTimeout(None) - # no need to call iter, we get a generator - maybeDeferred( - self.mbox.fetch_flags, messages, uid=uid - ).addCallback( - cbFetch, tag, query, uid - ).addErrback(ebFetch, tag) - elif len(query) == 1 and str(query[0]) == "rfc822.header": - self._oldTimeout = self.setTimeout(None) - # no need to call iter, we get a generator - maybeDeferred( - self.mbox.fetch_headers, messages, uid=uid - ).addCallback( - cbFetch, tag, query, uid - ).addErrback(ebFetch, tag) - else: - self._oldTimeout = self.setTimeout(None) - # no need to call iter, we get a generator - maybeDeferred( - self.mbox.fetch, messages, uid=uid - ).addCallback( - cbFetch, tag, query, uid - ).addErrback( - ebFetch, tag) - - deferLater(reactor, - 2, self.mbox.unset_recent_flags, messages) - deferLater(reactor, 1, self.mbox.signal_unread_to_ui) - - select_FETCH = (do_FETCH, imap4.IMAP4Server.arg_seqset, - imap4.IMAP4Server.arg_fetchatt) - - def do_COPY(self, tag, messages, mailbox, uid=0): - from twisted.internet import reactor - imap4.IMAP4Server.do_COPY(self, tag, messages, mailbox, uid) - deferLater(reactor, - 2, self.mbox.unset_recent_flags, messages) - deferLater(reactor, 1, self.mbox.signal_unread_to_ui) - - select_COPY = (do_COPY, imap4.IMAP4Server.arg_seqset, - imap4.IMAP4Server.arg_astring) - - def notifyNew(self, ignored): - """ - Notify new messages to listeners. - """ - self.mbox.notify_new() - - def _cbSelectWork(self, mbox, cmdName, tag): - """ - Callback for selectWork, patched to avoid conformance errors due to - incomplete UIDVALIDITY line. - """ - if mbox is None: - self.sendNegativeResponse(tag, 'No such mailbox') - return - if '\\noselect' in [s.lower() for s in mbox.getFlags()]: - self.sendNegativeResponse(tag, 'Mailbox cannot be selected') - return - - flags = mbox.getFlags() - self.sendUntaggedResponse(str(mbox.getMessageCount()) + ' EXISTS') - self.sendUntaggedResponse(str(mbox.getRecentCount()) + ' RECENT') - self.sendUntaggedResponse('FLAGS (%s)' % ' '.join(flags)) - - # Patched ------------------------------------------------------- - # imaptest was complaining about the incomplete line, we're adding - # "UIDs valid" here. - self.sendPositiveResponse( - None, '[UIDVALIDITY %d] UIDs valid' % mbox.getUIDValidity()) - # ---------------------------------------------------------------- - - s = mbox.isWriteable() and 'READ-WRITE' or 'READ-ONLY' - mbox.addListener(self) - self.sendPositiveResponse(tag, '[%s] %s successful' % (s, cmdName)) - self.state = 'select' - self.mbox = mbox - - class IMAPAuthRealm(object): """ Dummy authentication realm. Do not use in production! @@ -288,6 +125,8 @@ def run_service(*args, **kwargs): the reactor when starts listening, and the factory for the protocol. """ + from twisted.internet import reactor + leap_assert(len(args) == 2) soledad, keymanager = args leap_assert_type(soledad, Soledad) @@ -302,8 +141,6 @@ def run_service(*args, **kwargs): uuid = soledad._get_uuid() factory = LeapIMAPFactory(uuid, userid, soledad) - from twisted.internet import reactor - try: tport = reactor.listenTCP(port, factory, interface="localhost") @@ -330,14 +167,3 @@ def run_service(*args, **kwargs): # not ok, signal error. leap_events.signal(IMAP_SERVICE_FAILED_TO_START, str(port)) - - def checkpoint(self): - """ - Called when the client issues a CHECK command. - - This should perform any checkpoint operations required by the server. - It may be a long running operation, but may not block. If it returns - a deferred, the client will only be informed of success (or failure) - when the deferred's callback (or errback) is invoked. - """ - return None -- cgit v1.2.3 From e2218eec4fd91e4648160a05e3debc05efa0d0d9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 23 Jan 2014 02:36:38 -0400 Subject: add soledadstore class move parts-related bits to messageparts pass soledad in initialization for memory messages --- src/leap/mail/imap/service/imap.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 3f99da6..8350988 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -32,6 +32,7 @@ from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail from leap.mail.imap.memorystore import MemoryStore from leap.mail.imap.server import LeapIMAPServer +from leap.mail.imap.soledadstore import SoledadStore from leap.soledad.client import Soledad # The default port in which imap service will run @@ -96,7 +97,8 @@ class LeapIMAPFactory(ServerFactory): self._uuid = uuid self._userid = userid self._soledad = soledad - self._memstore = MemoryStore() + self._memstore = MemoryStore( + permanent_store=SoledadStore(soledad)) theAccount = SoledadBackedAccount( uuid, soledad=soledad, -- cgit v1.2.3 From 3a8fda3aa4645adbba228e7d2f204bfe6d400321 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 31 Jan 2014 22:16:26 -0400 Subject: enable manhole for debugging --- src/leap/mail/imap/service/imap.py | 125 +++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 8350988..8b95f75 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -18,6 +18,7 @@ Imap service initialization """ import logging +import os from twisted.internet.protocol import ServerFactory from twisted.internet.error import CannotListenError @@ -64,6 +65,8 @@ except Exception: ###################################################### +DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) + class IMAPAuthRealm(object): """ @@ -118,6 +121,118 @@ class LeapIMAPFactory(ServerFactory): return imapProtocol +MANHOLE_PORT = 2222 + + +def getManholeFactory(namespace, user, secret): + """ + Get an administrative manhole into the application. + + :param namespace: the namespace to show in the manhole + :type namespace: dict + :param user: the user to authenticate into the administrative shell. + :type user: str + :param secret: pass for this manhole + :type secret: str + """ + import string + + from twisted.cred.portal import Portal + from twisted.conch import manhole, manhole_ssh + from twisted.conch.insults import insults + from twisted.cred.checkers import ( + InMemoryUsernamePasswordDatabaseDontUse as MemoryDB) + + from rlcompleter import Completer + + class EnhancedColoredManhole(manhole.ColoredManhole): + """ + A Manhole with some primitive autocomplete support. + """ + # TODO use introspection to make life easier + + def find_common(self, l): + """ + find common parts in thelist items + ex: 'ab' for ['abcd','abce','abf'] + requires an ordered list + """ + if len(l) == 1: + return l[0] + + init = l[0] + for item in l[1:]: + for i, (x, y) in enumerate(zip(init, item)): + if x != y: + init = "".join(init[:i]) + break + + if not init: + return None + return init + + def handle_TAB(self): + """ + Trap the TAB keystroke + """ + necessarypart = "".join(self.lineBuffer).split(' ')[-1] + completer = Completer(globals()) + if completer.complete(necessarypart, 0): + matches = list(set(completer.matches)) # has multiples + + if len(matches) == 1: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(matches[0]) + self.lineBufferIndex = len(self.lineBuffer) + else: + matches.sort() + commons = self.find_common(matches) + if commons: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(commons) + self.lineBufferIndex = len(self.lineBuffer) + + self.terminal.nextLine() + while matches: + matches, part = matches[4:], matches[:4] + for item in part: + self.terminal.write('%s' % item.ljust(30)) + self.terminal.write('\n') + self.terminal.nextLine() + + self.terminal.eraseLine() + self.terminal.cursorBackward(self.lineBufferIndex + 5) + self.terminal.write("%s %s" % ( + self.ps[self.pn], "".join(self.lineBuffer))) + + def keystrokeReceived(self, keyID, modifier): + """ + Act upon any keystroke received. + """ + self.keyHandlers.update({'\b': self.handle_BACKSPACE}) + m = self.keyHandlers.get(keyID) + if m is not None: + m() + elif keyID in string.printable: + self.characterReceived(keyID, False) + + sshRealm = manhole_ssh.TerminalRealm() + + def chainedProtocolFactory(): + return insults.ServerProtocol(EnhancedColoredManhole, namespace) + + sshRealm = manhole_ssh.TerminalRealm() + sshRealm.chainedProtocolFactory = chainedProtocolFactory + + portal = Portal( + sshRealm, [MemoryDB(**{user: secret})]) + + f = manhole_ssh.ConchFactory(portal) + return f + + def run_service(*args, **kwargs): """ Main entry point to run the service from the client. @@ -163,6 +278,16 @@ def run_service(*args, **kwargs): else: # all good. # (the caller has still to call fetcher.start_loop) + + if DO_MANHOLE: + # TODO get pass from env var.too. + manhole_factory = getManholeFactory( + {'f': factory, + 'a': factory.theAccount, + 'gm': factory.theAccount.getMailbox}, + "boss", "leap") + reactor.listenTCP(MANHOLE_PORT, manhole_factory, + interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) return fetcher, tport, factory -- cgit v1.2.3 From 23e28bae2c3cb74e00e29ee8add0b73adeb65c2b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 4 Feb 2014 10:57:49 -0400 Subject: fixes after review * Some more docstring completion/fixes. * Removed unneeded str coertion. * Handle mailbox name in logs. * Separate manhole boilerplate into its own file. --- src/leap/mail/imap/service/imap.py | 118 ++---------------------------- src/leap/mail/imap/service/manhole.py | 130 ++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 114 deletions(-) create mode 100644 src/leap/mail/imap/service/manhole.py (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 8b95f75..5487cfc 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -66,6 +66,8 @@ except Exception: ###################################################### DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) +if DO_MANHOLE: + from leap.mail.imap.service import manhole class IMAPAuthRealm(object): @@ -121,118 +123,6 @@ class LeapIMAPFactory(ServerFactory): return imapProtocol -MANHOLE_PORT = 2222 - - -def getManholeFactory(namespace, user, secret): - """ - Get an administrative manhole into the application. - - :param namespace: the namespace to show in the manhole - :type namespace: dict - :param user: the user to authenticate into the administrative shell. - :type user: str - :param secret: pass for this manhole - :type secret: str - """ - import string - - from twisted.cred.portal import Portal - from twisted.conch import manhole, manhole_ssh - from twisted.conch.insults import insults - from twisted.cred.checkers import ( - InMemoryUsernamePasswordDatabaseDontUse as MemoryDB) - - from rlcompleter import Completer - - class EnhancedColoredManhole(manhole.ColoredManhole): - """ - A Manhole with some primitive autocomplete support. - """ - # TODO use introspection to make life easier - - def find_common(self, l): - """ - find common parts in thelist items - ex: 'ab' for ['abcd','abce','abf'] - requires an ordered list - """ - if len(l) == 1: - return l[0] - - init = l[0] - for item in l[1:]: - for i, (x, y) in enumerate(zip(init, item)): - if x != y: - init = "".join(init[:i]) - break - - if not init: - return None - return init - - def handle_TAB(self): - """ - Trap the TAB keystroke - """ - necessarypart = "".join(self.lineBuffer).split(' ')[-1] - completer = Completer(globals()) - if completer.complete(necessarypart, 0): - matches = list(set(completer.matches)) # has multiples - - if len(matches) == 1: - length = len(necessarypart) - self.lineBuffer = self.lineBuffer[:-length] - self.lineBuffer.extend(matches[0]) - self.lineBufferIndex = len(self.lineBuffer) - else: - matches.sort() - commons = self.find_common(matches) - if commons: - length = len(necessarypart) - self.lineBuffer = self.lineBuffer[:-length] - self.lineBuffer.extend(commons) - self.lineBufferIndex = len(self.lineBuffer) - - self.terminal.nextLine() - while matches: - matches, part = matches[4:], matches[:4] - for item in part: - self.terminal.write('%s' % item.ljust(30)) - self.terminal.write('\n') - self.terminal.nextLine() - - self.terminal.eraseLine() - self.terminal.cursorBackward(self.lineBufferIndex + 5) - self.terminal.write("%s %s" % ( - self.ps[self.pn], "".join(self.lineBuffer))) - - def keystrokeReceived(self, keyID, modifier): - """ - Act upon any keystroke received. - """ - self.keyHandlers.update({'\b': self.handle_BACKSPACE}) - m = self.keyHandlers.get(keyID) - if m is not None: - m() - elif keyID in string.printable: - self.characterReceived(keyID, False) - - sshRealm = manhole_ssh.TerminalRealm() - - def chainedProtocolFactory(): - return insults.ServerProtocol(EnhancedColoredManhole, namespace) - - sshRealm = manhole_ssh.TerminalRealm() - sshRealm.chainedProtocolFactory = chainedProtocolFactory - - portal = Portal( - sshRealm, [MemoryDB(**{user: secret})]) - - f = manhole_ssh.ConchFactory(portal) - return f - - def run_service(*args, **kwargs): """ Main entry point to run the service from the client. @@ -281,12 +171,12 @@ def run_service(*args, **kwargs): if DO_MANHOLE: # TODO get pass from env var.too. - manhole_factory = getManholeFactory( + manhole_factory = manhole.getManholeFactory( {'f': factory, 'a': factory.theAccount, 'gm': factory.theAccount.getMailbox}, "boss", "leap") - reactor.listenTCP(MANHOLE_PORT, manhole_factory, + reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) diff --git a/src/leap/mail/imap/service/manhole.py b/src/leap/mail/imap/service/manhole.py new file mode 100644 index 0000000..c83ae89 --- /dev/null +++ b/src/leap/mail/imap/service/manhole.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# manhole.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Utilities for enabling the manhole administrative interface into the +LEAP Mail application. +""" +MANHOLE_PORT = 2222 + + +def getManholeFactory(namespace, user, secret): + """ + Get an administrative manhole into the application. + + :param namespace: the namespace to show in the manhole + :type namespace: dict + :param user: the user to authenticate into the administrative shell. + :type user: str + :param secret: pass for this manhole + :type secret: str + """ + import string + + from twisted.cred.portal import Portal + from twisted.conch import manhole, manhole_ssh + from twisted.conch.insults import insults + from twisted.cred.checkers import ( + InMemoryUsernamePasswordDatabaseDontUse as MemoryDB) + + from rlcompleter import Completer + + class EnhancedColoredManhole(manhole.ColoredManhole): + """ + A Manhole with some primitive autocomplete support. + """ + # TODO use introspection to make life easier + + def find_common(self, l): + """ + find common parts in thelist items + ex: 'ab' for ['abcd','abce','abf'] + requires an ordered list + """ + if len(l) == 1: + return l[0] + + init = l[0] + for item in l[1:]: + for i, (x, y) in enumerate(zip(init, item)): + if x != y: + init = "".join(init[:i]) + break + + if not init: + return None + return init + + def handle_TAB(self): + """ + Trap the TAB keystroke. + """ + necessarypart = "".join(self.lineBuffer).split(' ')[-1] + completer = Completer(globals()) + if completer.complete(necessarypart, 0): + matches = list(set(completer.matches)) # has multiples + + if len(matches) == 1: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(matches[0]) + self.lineBufferIndex = len(self.lineBuffer) + else: + matches.sort() + commons = self.find_common(matches) + if commons: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(commons) + self.lineBufferIndex = len(self.lineBuffer) + + self.terminal.nextLine() + while matches: + matches, part = matches[4:], matches[:4] + for item in part: + self.terminal.write('%s' % item.ljust(30)) + self.terminal.write('\n') + self.terminal.nextLine() + + self.terminal.eraseLine() + self.terminal.cursorBackward(self.lineBufferIndex + 5) + self.terminal.write("%s %s" % ( + self.ps[self.pn], "".join(self.lineBuffer))) + + def keystrokeReceived(self, keyID, modifier): + """ + Act upon any keystroke received. + """ + self.keyHandlers.update({'\b': self.handle_BACKSPACE}) + m = self.keyHandlers.get(keyID) + if m is not None: + m() + elif keyID in string.printable: + self.characterReceived(keyID, False) + + sshRealm = manhole_ssh.TerminalRealm() + + def chainedProtocolFactory(): + return insults.ServerProtocol(EnhancedColoredManhole, namespace) + + sshRealm = manhole_ssh.TerminalRealm() + sshRealm.chainedProtocolFactory = chainedProtocolFactory + + portal = Portal( + sshRealm, [MemoryDB(**{user: secret})]) + + f = manhole_ssh.ConchFactory(portal) + return f -- cgit v1.2.3 From 362aaec0897261973e58b4282f5c054985d1f113 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 6 Feb 2014 15:46:01 -0200 Subject: Flush IMAP data to disk when stopping. Closes #5095. --- src/leap/mail/imap/service/imap.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 5487cfc..93df51d 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -19,7 +19,9 @@ Imap service initialization """ import logging import os +import time +from twisted.internet import defer, threads from twisted.internet.protocol import ServerFactory from twisted.internet.error import CannotListenError from twisted.mail import imap4 @@ -122,6 +124,35 @@ class LeapIMAPFactory(ServerFactory): imapProtocol.factory = self return imapProtocol + def doStop(self, cv): + """ + Stops imap service (fetcher, factory and port). + + :param cv: A condition variable to which we can signal when imap + indeed stops. + :type cv: threading.Condition + :return: a Deferred that stops and flushes the in memory store data to + disk in another thread. + :rtype: Deferred + """ + ServerFactory.doStop(self) + + def _stop_imap_cb(): + logger.debug('Stopping in memory store.') + self._memstore.stop_and_flush() + while not self._memstore.producer.is_queue_empty(): + logger.debug('Waiting for queue to be empty.') + # TODO use a gatherResults over the new/dirty deferred list, + # as in memorystore's expunge() method. + time.sleep(1) + # notify that service has stopped + logger.debug('Notifying that service has stopped.') + cv.acquire() + cv.notify() + cv.release() + + return threads.deferToThread(_stop_imap_cb) + def run_service(*args, **kwargs): """ -- cgit v1.2.3 From ff3a6a640fdb345449a5f9cd3379bbaefa36111e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 6 Feb 2014 15:46:17 -0400 Subject: take recent count from memstore --- src/leap/mail/imap/service/imap.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 93df51d..726049c 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -115,7 +115,12 @@ class LeapIMAPFactory(ServerFactory): # XXX how to pass the store along? def buildProtocol(self, addr): - "Return a protocol suitable for the job." + """ + Return a protocol suitable for the job. + + :param addr: ??? + :type addr: ??? + """ imapProtocol = LeapIMAPServer( uuid=self._uuid, userid=self._userid, -- cgit v1.2.3 From 080c9207c70572a02f33d50697f39878713ab6ac Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 01:41:05 -0400 Subject: make the condition optional --- src/leap/mail/imap/service/imap.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 726049c..6041961 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -129,7 +129,7 @@ class LeapIMAPFactory(ServerFactory): imapProtocol.factory = self return imapProtocol - def doStop(self, cv): + def doStop(self, cv=None): """ Stops imap service (fetcher, factory and port). @@ -142,21 +142,23 @@ class LeapIMAPFactory(ServerFactory): """ ServerFactory.doStop(self) - def _stop_imap_cb(): - logger.debug('Stopping in memory store.') - self._memstore.stop_and_flush() - while not self._memstore.producer.is_queue_empty(): - logger.debug('Waiting for queue to be empty.') - # TODO use a gatherResults over the new/dirty deferred list, - # as in memorystore's expunge() method. - time.sleep(1) - # notify that service has stopped - logger.debug('Notifying that service has stopped.') - cv.acquire() - cv.notify() - cv.release() - - return threads.deferToThread(_stop_imap_cb) + if cv is not None: + def _stop_imap_cb(): + logger.debug('Stopping in memory store.') + self._memstore.stop_and_flush() + while not self._memstore.producer.is_queue_empty(): + logger.debug('Waiting for queue to be empty.') + # TODO use a gatherResults over the new/dirty + # deferred list, + # as in memorystore's expunge() method. + time.sleep(1) + # notify that service has stopped + logger.debug('Notifying that service has stopped.') + cv.acquire() + cv.notify() + cv.release() + + return threads.deferToThread(_stop_imap_cb) def run_service(*args, **kwargs): -- cgit v1.2.3 From 1217be6c792d87134f6801591c7bfa9536c9a3d1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 12 Feb 2014 12:40:04 -0400 Subject: suggest bigger threadpool to reactors that honor it --- src/leap/mail/imap/service/imap.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 6041961..a7799ca 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -171,6 +171,9 @@ def run_service(*args, **kwargs): the protocol. """ from twisted.internet import reactor + # it looks like qtreactor does not honor this, + # but other reactors should. + reactor.suggestThreadPoolSize(20) leap_assert(len(args) == 2) soledad, keymanager = args -- cgit v1.2.3 From 45733a231128cc06e123f352b4eb9886d6820878 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 14 Feb 2014 12:41:58 -0400 Subject: docstring fixes --- src/leap/mail/imap/service/imap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index a7799ca..b79d42d 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -118,8 +118,8 @@ class LeapIMAPFactory(ServerFactory): """ Return a protocol suitable for the job. - :param addr: ??? - :type addr: ??? + :param addr: remote ip address + :type addr: str """ imapProtocol = LeapIMAPServer( uuid=self._uuid, -- cgit v1.2.3 From f67aabeb382592f3d7d597acb0389b39a353d8b8 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 14 Feb 2014 12:42:58 -0400 Subject: add cProfiler instrumentation --- src/leap/mail/imap/service/imap.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index b79d42d..1175cdc 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -25,6 +25,7 @@ from twisted.internet import defer, threads from twisted.internet.protocol import ServerFactory from twisted.internet.error import CannotListenError from twisted.mail import imap4 +from twisted.python import log logger = logging.getLogger(__name__) @@ -71,6 +72,15 @@ DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) if DO_MANHOLE: from leap.mail.imap.service import manhole +DO_PROFILE = os.environ.get("LEAP_PROFILE", None) +if DO_PROFILE: + import cProfile + log.msg("Starting PROFILING...") + + PROFILE_DAT = "/tmp/leap_mail_profile.pstats" + pr = cProfile.Profile() + pr.enable() + class IMAPAuthRealm(object): """ @@ -140,6 +150,11 @@ class LeapIMAPFactory(ServerFactory): disk in another thread. :rtype: Deferred """ + if DO_PROFILE: + log.msg("Stopping PROFILING") + pr.disable() + pr.dump_stats(PROFILE_DAT) + ServerFactory.doStop(self) if cv is not None: -- cgit v1.2.3 From ce9185be85d82d95799dfb0644d4f363feeeeeae Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Mon, 17 Feb 2014 18:46:12 -0300 Subject: Update keymanager kwargs, related to #5120. --- src/leap/mail/imap/service/imap-server.tac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index b65bb17..feeca06 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -132,7 +132,7 @@ tempdir = "/tmp/" soledad = initialize_soledad(uuid, userid, passwd, secrets, localdb, gnupg_home, tempdir) km_args = (userid, "https://localhost", soledad) km_kwargs = { - "session_id": "", + "token": "", "ca_cert_path": "", "api_uri": "", "api_version": "", -- cgit v1.2.3 From c8b2163c8d8063f5e9a5cc7a2a721e933f002be0 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Mon, 17 Feb 2014 18:50:53 -0300 Subject: pep8 fixes. --- src/leap/mail/imap/service/imap-server.tac | 33 ++++++++++++++++-------------- 1 file changed, 18 insertions(+), 15 deletions(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index feeca06..651f71b 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# imap-server.tac +# imap-server.tac # Copyright (C) 2013,2014 LEAP # # This program is free software: you can redistribute it and/or modify @@ -97,8 +97,10 @@ print "[+] Running LEAP IMAP Service" bmconf = os.environ.get("LEAP_MAIL_CONF", "") if not bmconf: - print "[-] Please set LEAP_MAIL_CONF environment variable pointing to your config." - sys.exit(1) + print ("[-] Please set LEAP_MAIL_CONF environment variable " + "pointing to your config.") + sys.exit(1) + SECTION = "leap_mail" cp = ConfigParser.ConfigParser() cp.read(bmconf) @@ -111,11 +113,11 @@ passwd = unicode(cp.get(SECTION, "passwd")) port = 1984 if not userid or not uuid: - print "[-] Config file missing userid or uuid field" - sys.exit(1) + print "[-] Config file missing userid or uuid field" + sys.exit(1) if not passwd: - passwd = unicode(getpass.getpass("Soledad passphrase: ")) + passwd = unicode(getpass.getpass("Soledad passphrase: ")) secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,)) @@ -129,16 +131,17 @@ tempdir = "/tmp/" # Ad-hoc soledad/keymanager initialization. -soledad = initialize_soledad(uuid, userid, passwd, secrets, localdb, gnupg_home, tempdir) +soledad = initialize_soledad(uuid, userid, passwd, secrets, + localdb, gnupg_home, tempdir) km_args = (userid, "https://localhost", soledad) -km_kwargs = { - "token": "", - "ca_cert_path": "", - "api_uri": "", - "api_version": "", - "uid": uuid, - "gpgbinary": "/usr/bin/gpg" -} +km_kwargs = { + "token": "", + "ca_cert_path": "", + "api_uri": "", + "api_version": "", + "uid": uuid, + "gpgbinary": "/usr/bin/gpg" +} keymanager = KeyManager(*km_args, **km_kwargs) ################################################## -- cgit v1.2.3 From 38863f978437fcd33254b830e7d4906f4fbeccdc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 5 Mar 2014 12:13:43 -0400 Subject: workaround attempt for the recursionlimit bug with qtreactor. Increasing the recursion limit by an order of magnitude here seems to allow a fetch of a mailbox with 500 mails. See #5196 for discussion of alternatives. --- src/leap/mail/imap/service/imap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/mail/imap/service') diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 1175cdc..10ba32a 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -57,7 +57,7 @@ import resource import sys try: - sys.setrecursionlimit(10**6) + sys.setrecursionlimit(10**7) except Exception: print "Error setting recursion limit" try: -- cgit v1.2.3