diff options
Diffstat (limited to 'src/leap/mail/imap/service')
| -rw-r--r-- | src/leap/mail/imap/service/imap-server.tac | 185 | ||||
| -rw-r--r-- | src/leap/mail/imap/service/imap.py | 185 | ||||
| -rw-r--r-- | src/leap/mail/imap/service/manhole.py | 130 |
3 files changed, 375 insertions, 125 deletions
diff --git a/src/leap/mail/imap/service/imap-server.tac b/src/leap/mail/imap/service/imap-server.tac index da72cae..651f71b 100644 --- a/src/leap/mail/imap/service/imap-server.tac +++ b/src/leap/mail/imap/service/imap-server.tac @@ -1,69 +1,160 @@ +# -*- coding: utf-8 -*- +# imap-server.tac +# Copyright (C) 2013,2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +TAC file for initialization of the imap service using twistd. + +Use this for debugging and testing the imap server using a native reactor. + +For now, and for debugging/testing purposes, you need +to pass a config file with the following structure: + +[leap_mail] +userid = "user@provider" +uuid = "deadbeefdeadabad" +passwd = "supersecret" # optional, will get prompted if not found. +""" import ConfigParser +import getpass import os +import sys -from leap.soledad.client import Soledad +from leap.keymanager import KeyManager from leap.mail.imap.service import imap -from leap.common.config import get_path_prefix - - -config = ConfigParser.ConfigParser() -config.read([os.path.expanduser('~/.config/leap/mail/mail.conf')]) - -userID = config.get('mail', 'address') -privkey = open(os.path.expanduser('~/.config/leap/mail/privkey')).read() -nickserver_url = "" +from leap.soledad.client import Soledad -d = {} +from twisted.application import service, internet -for key in ('uid', 'passphrase', 'server', 'pemfile', 'token'): - d[key] = config.get('mail', key) +# TODO should get this initializers from some authoritative mocked source +# We might want to put them the soledad itself. -def initialize_soledad_mailbox(user_uuid, soledad_pass, server_url, - server_pemfile, token): +def initialize_soledad(uuid, email, passwd, + secrets, localdb, + gnupg_home, tempdir): """ Initializes soledad by hand - :param user_uuid: - :param soledad_pass: - :param server_url: - :param server_pemfile: - :param token: - + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir :rtype: Soledad instance """ + # XXX TODO unify with an authoritative source of mocks + # for soledad (or partial initializations). + # This is copied from the imap tests. + + server_url = "http://provider" + cert_file = "" - base_config = get_path_prefix() + class Mock(object): + def __init__(self, return_value=None): + self._return = return_value - secret_path = os.path.join( - base_config, "leap", "soledad", "%s.secret" % user_uuid) - soledad_path = os.path.join( - base_config, "leap", "soledad", "%s-mailbox.db" % user_uuid) + def __call__(self, *args, **kwargs): + return self._return - _soledad = Soledad( - user_uuid, - soledad_pass, - secret_path, - soledad_path, + class MockSharedDB(object): + + get_doc = Mock() + put_doc = Mock() + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) + + def __call__(self): + return self + + Soledad._shared_db = MockSharedDB() + soledad = Soledad( + uuid, + passwd, + secrets, + localdb, server_url, - server_pemfile, - token) + cert_file) + + return soledad + +###################################################################### +# Remember to set your config files, see module documentation above! +###################################################################### + +print "[+] Running LEAP IMAP Service" + + +bmconf = os.environ.get("LEAP_MAIL_CONF", "") +if not bmconf: + print ("[-] Please set LEAP_MAIL_CONF environment variable " + "pointing to your config.") + sys.exit(1) + +SECTION = "leap_mail" +cp = ConfigParser.ConfigParser() +cp.read(bmconf) + +userid = cp.get(SECTION, "userid") +uuid = cp.get(SECTION, "uuid") +passwd = unicode(cp.get(SECTION, "passwd")) + +# XXX get this right from the environment variable !!! +port = 1984 + +if not userid or not uuid: + print "[-] Config file missing userid or uuid field" + sys.exit(1) + +if not passwd: + passwd = unicode(getpass.getpass("Soledad passphrase: ")) + + +secrets = os.path.expanduser("~/.config/leap/soledad/%s.secret" % (uuid,)) +localdb = os.path.expanduser("~/.config/leap/soledad/%s.db" % (uuid,)) + +# XXX Is this really used? Should point it to user var dirs defined in xdg? +gnupg_home = "/tmp/" +tempdir = "/tmp/" + +################################################### + +# Ad-hoc soledad/keymanager initialization. + +soledad = initialize_soledad(uuid, userid, passwd, secrets, + localdb, gnupg_home, tempdir) +km_args = (userid, "https://localhost", soledad) +km_kwargs = { + "token": "", + "ca_cert_path": "", + "api_uri": "", + "api_version": "", + "uid": uuid, + "gpgbinary": "/usr/bin/gpg" +} +keymanager = KeyManager(*km_args, **km_kwargs) + +################################################## - return _soledad +# Ok, let's expose the application object for the twistd application +# framework to pick up from here... -soledad = initialize_soledad_mailbox( - d['uid'], - d['passphrase'], - d['server'], - d['pemfile'], - d['token']) -# import the private key ---- should sync it from remote! -from leap.common.keymanager.openpgp import OpenPGPScheme -opgp = OpenPGPScheme(soledad) -opgp.put_ascii_key(privkey) +def getIMAPService(): + factory = imap.LeapIMAPFactory(uuid, userid, soledad) + return internet.TCPServer(port, factory, interface="localhost") -from leap.common.keymanager import KeyManager -keymanager = KeyManager(userID, nickserver_url, soledad, d['token']) -imap.run_service(soledad, keymanager) +application = service.Application("LEAP IMAP Application") +service = getIMAPService() +service.setServiceParent(application) diff --git a/src/leap/mail/imap/service/imap.py b/src/leap/mail/imap/service/imap.py index 8756ddc..10ba32a 100644 --- a/src/leap/mail/imap/service/imap.py +++ b/src/leap/mail/imap/service/imap.py @@ -17,23 +17,26 @@ """ Imap service initialization """ -from copy import copy - import logging +import os +import time +from twisted.internet import defer, threads from twisted.internet.protocol import ServerFactory from twisted.internet.error import CannotListenError from twisted.mail import imap4 from twisted.python import log -from twisted import cred logger = logging.getLogger(__name__) from leap.common import events as leap_events from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.keymanager import KeyManager -from leap.mail.imap.server import SoledadBackedAccount +from leap.mail.imap.account import SoledadBackedAccount from leap.mail.imap.fetch import LeapIncomingMail +from leap.mail.imap.memorystore import MemoryStore +from leap.mail.imap.server import LeapIMAPServer +from leap.mail.imap.soledadstore import SoledadStore from leap.soledad.client import Soledad # The default port in which imap service will run @@ -45,75 +48,38 @@ INCOMING_CHECK_PERIOD = 60 from leap.common.events.events_pb2 import IMAP_SERVICE_STARTED from leap.common.events.events_pb2 import IMAP_SERVICE_FAILED_TO_START -from leap.common.events.events_pb2 import IMAP_CLIENT_LOGIN - - -class LeapIMAPServer(imap4.IMAP4Server): - """ - An IMAP4 Server with mailboxes backed by soledad - """ - def __init__(self, *args, **kwargs): - # pop extraneous arguments - soledad = kwargs.pop('soledad', None) - uuid = kwargs.pop('uuid', None) - userid = kwargs.pop('userid', None) - leap_assert(soledad, "need a soledad instance") - leap_assert_type(soledad, Soledad) - leap_assert(uuid, "need a user in the initialization") - - self._userid = userid - # initialize imap server! - imap4.IMAP4Server.__init__(self, *args, **kwargs) +###################################################### +# Temporary workaround for RecursionLimit when using +# qt4reactor. Do remove when we move to poll or select +# reactor, which do not show those problems. See #4974 +import resource +import sys - # we should initialize the account here, - # but we move it to the factory so we can - # populate the test account properly (and only once - # per session) +try: + sys.setrecursionlimit(10**7) +except Exception: + print "Error setting recursion limit" +try: + # Increase max stack size from 8MB to 256MB + resource.setrlimit(resource.RLIMIT_STACK, (2**28, -1)) +except Exception: + print "Error setting stack size" - # theAccount = SoledadBackedAccount( - # user, soledad=soledad) +###################################################### - # --------------------------------- - # XXX pre-populate acct for tests!! - # populate_test_account(theAccount) - # --------------------------------- - #self.theAccount = theAccount +DO_MANHOLE = os.environ.get("LEAP_MAIL_MANHOLE", None) +if DO_MANHOLE: + from leap.mail.imap.service import manhole - def lineReceived(self, line): - """ - Attempt to parse a single line from the server. +DO_PROFILE = os.environ.get("LEAP_PROFILE", None) +if DO_PROFILE: + import cProfile + log.msg("Starting PROFILING...") - :param line: the line from the server, without the line delimiter. - :type line: str - """ - if "login" in line.lower(): - # avoid to log the pass, even though we are using a dummy auth - # by now. - msg = line[:7] + " [...]" - else: - msg = copy(line) - log.msg('rcv: %s' % msg) - imap4.IMAP4Server.lineReceived(self, line) - - def authenticateLogin(self, username, password): - """ - Lookup the account with the given parameters, and deny - the improper combinations. - - :param username: the username that is attempting authentication. - :type username: str - :param password: the password to authenticate with. - :type password: str - """ - # XXX this should use portal: - # return portal.login(cred.credentials.UsernamePassword(user, pass) - if username != self._userid: - # bad username, reject. - raise cred.error.UnauthorizedLogin() - # any dummy password is allowed so far. use realm instead! - leap_events.signal(IMAP_CLIENT_LOGIN, "1") - return imap4.IAccount, self.theAccount, lambda: None + PROFILE_DAT = "/tmp/leap_mail_profile.pstats" + pr = cProfile.Profile() + pr.enable() class IMAPAuthRealm(object): @@ -148,13 +114,23 @@ class LeapIMAPFactory(ServerFactory): self._uuid = uuid self._userid = userid self._soledad = soledad + self._memstore = MemoryStore( + permanent_store=SoledadStore(soledad)) theAccount = SoledadBackedAccount( - uuid, soledad=soledad) + uuid, soledad=soledad, + memstore=self._memstore) self.theAccount = theAccount + # XXX how to pass the store along? + def buildProtocol(self, addr): - "Return a protocol suitable for the job." + """ + Return a protocol suitable for the job. + + :param addr: remote ip address + :type addr: str + """ imapProtocol = LeapIMAPServer( uuid=self._uuid, userid=self._userid, @@ -163,6 +139,42 @@ class LeapIMAPFactory(ServerFactory): imapProtocol.factory = self return imapProtocol + def doStop(self, cv=None): + """ + Stops imap service (fetcher, factory and port). + + :param cv: A condition variable to which we can signal when imap + indeed stops. + :type cv: threading.Condition + :return: a Deferred that stops and flushes the in memory store data to + disk in another thread. + :rtype: Deferred + """ + if DO_PROFILE: + log.msg("Stopping PROFILING") + pr.disable() + pr.dump_stats(PROFILE_DAT) + + ServerFactory.doStop(self) + + if cv is not None: + def _stop_imap_cb(): + logger.debug('Stopping in memory store.') + self._memstore.stop_and_flush() + while not self._memstore.producer.is_queue_empty(): + logger.debug('Waiting for queue to be empty.') + # TODO use a gatherResults over the new/dirty + # deferred list, + # as in memorystore's expunge() method. + time.sleep(1) + # notify that service has stopped + logger.debug('Notifying that service has stopped.') + cv.acquire() + cv.notify() + cv.release() + + return threads.deferToThread(_stop_imap_cb) + def run_service(*args, **kwargs): """ @@ -173,6 +185,11 @@ def run_service(*args, **kwargs): the reactor when starts listening, and the factory for the protocol. """ + from twisted.internet import reactor + # it looks like qtreactor does not honor this, + # but other reactors should. + reactor.suggestThreadPoolSize(20) + leap_assert(len(args) == 2) soledad, keymanager = args leap_assert_type(soledad, Soledad) @@ -182,21 +199,23 @@ def run_service(*args, **kwargs): check_period = kwargs.get('check_period', INCOMING_CHECK_PERIOD) userid = kwargs.get('userid', None) leap_check(userid is not None, "need an user id") + offline = kwargs.get('offline', False) uuid = soledad._get_uuid() factory = LeapIMAPFactory(uuid, userid, soledad) - from twisted.internet import reactor - try: tport = reactor.listenTCP(port, factory, interface="localhost") - fetcher = LeapIncomingMail( - keymanager, - soledad, - factory.theAccount, - check_period, - userid) + if not offline: + fetcher = LeapIncomingMail( + keymanager, + soledad, + factory.theAccount, + check_period, + userid) + else: + fetcher = None except CannotListenError: logger.error("IMAP Service failed to start: " "cannot listen in port %s" % (port,)) @@ -204,7 +223,17 @@ def run_service(*args, **kwargs): logger.error("Error launching IMAP service: %r" % (exc,)) else: # all good. - fetcher.start_loop() + # (the caller has still to call fetcher.start_loop) + + if DO_MANHOLE: + # TODO get pass from env var.too. + manhole_factory = manhole.getManholeFactory( + {'f': factory, + 'a': factory.theAccount, + 'gm': factory.theAccount.getMailbox}, + "boss", "leap") + reactor.listenTCP(manhole.MANHOLE_PORT, manhole_factory, + interface="127.0.0.1") logger.debug("IMAP4 Server is RUNNING in port %s" % (port,)) leap_events.signal(IMAP_SERVICE_STARTED, str(port)) return fetcher, tport, factory diff --git a/src/leap/mail/imap/service/manhole.py b/src/leap/mail/imap/service/manhole.py new file mode 100644 index 0000000..c83ae89 --- /dev/null +++ b/src/leap/mail/imap/service/manhole.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# manhole.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Utilities for enabling the manhole administrative interface into the +LEAP Mail application. +""" +MANHOLE_PORT = 2222 + + +def getManholeFactory(namespace, user, secret): + """ + Get an administrative manhole into the application. + + :param namespace: the namespace to show in the manhole + :type namespace: dict + :param user: the user to authenticate into the administrative shell. + :type user: str + :param secret: pass for this manhole + :type secret: str + """ + import string + + from twisted.cred.portal import Portal + from twisted.conch import manhole, manhole_ssh + from twisted.conch.insults import insults + from twisted.cred.checkers import ( + InMemoryUsernamePasswordDatabaseDontUse as MemoryDB) + + from rlcompleter import Completer + + class EnhancedColoredManhole(manhole.ColoredManhole): + """ + A Manhole with some primitive autocomplete support. + """ + # TODO use introspection to make life easier + + def find_common(self, l): + """ + find common parts in thelist items + ex: 'ab' for ['abcd','abce','abf'] + requires an ordered list + """ + if len(l) == 1: + return l[0] + + init = l[0] + for item in l[1:]: + for i, (x, y) in enumerate(zip(init, item)): + if x != y: + init = "".join(init[:i]) + break + + if not init: + return None + return init + + def handle_TAB(self): + """ + Trap the TAB keystroke. + """ + necessarypart = "".join(self.lineBuffer).split(' ')[-1] + completer = Completer(globals()) + if completer.complete(necessarypart, 0): + matches = list(set(completer.matches)) # has multiples + + if len(matches) == 1: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(matches[0]) + self.lineBufferIndex = len(self.lineBuffer) + else: + matches.sort() + commons = self.find_common(matches) + if commons: + length = len(necessarypart) + self.lineBuffer = self.lineBuffer[:-length] + self.lineBuffer.extend(commons) + self.lineBufferIndex = len(self.lineBuffer) + + self.terminal.nextLine() + while matches: + matches, part = matches[4:], matches[:4] + for item in part: + self.terminal.write('%s' % item.ljust(30)) + self.terminal.write('\n') + self.terminal.nextLine() + + self.terminal.eraseLine() + self.terminal.cursorBackward(self.lineBufferIndex + 5) + self.terminal.write("%s %s" % ( + self.ps[self.pn], "".join(self.lineBuffer))) + + def keystrokeReceived(self, keyID, modifier): + """ + Act upon any keystroke received. + """ + self.keyHandlers.update({'\b': self.handle_BACKSPACE}) + m = self.keyHandlers.get(keyID) + if m is not None: + m() + elif keyID in string.printable: + self.characterReceived(keyID, False) + + sshRealm = manhole_ssh.TerminalRealm() + + def chainedProtocolFactory(): + return insults.ServerProtocol(EnhancedColoredManhole, namespace) + + sshRealm = manhole_ssh.TerminalRealm() + sshRealm.chainedProtocolFactory = chainedProtocolFactory + + portal = Portal( + sshRealm, [MemoryDB(**{user: secret})]) + + f = manhole_ssh.ConchFactory(portal) + return f |