diff options
Diffstat (limited to 'src/leap/bitmask/services')
18 files changed, 421 insertions, 351 deletions
diff --git a/src/leap/bitmask/services/__init__.py b/src/leap/bitmask/services/__init__.py index ba12ba4e..54426669 100644 --- a/src/leap/bitmask/services/__init__.py +++ b/src/leap/bitmask/services/__init__.py @@ -17,7 +17,6 @@ """ Services module. """ -import logging import os import sys @@ -25,6 +24,7 @@ from PySide import QtCore from leap.bitmask.config import flags from leap.bitmask.crypto.srpauth import SRPAuth +from leap.bitmask.logs.utils import get_logger from leap.bitmask.util.constants import REQUEST_TIMEOUT from leap.bitmask.util.privilege_policies import is_missing_policy_permissions from leap.bitmask.util.request_helpers import get_content @@ -34,7 +34,7 @@ from leap.common.check import leap_assert from leap.common.config.baseconfig import BaseConfig from leap.common.files import get_mtime -logger = logging.getLogger(__name__) +logger = get_logger() EIP_SERVICE = u"openvpn" diff --git a/src/leap/bitmask/services/abstractbootstrapper.py b/src/leap/bitmask/services/abstractbootstrapper.py index 77929b75..191309ba 100644 --- a/src/leap/bitmask/services/abstractbootstrapper.py +++ b/src/leap/bitmask/services/abstractbootstrapper.py @@ -18,8 +18,6 @@ """ Abstract bootstrapper implementation """ -import logging - import requests from functools import partial @@ -27,12 +25,13 @@ from functools import partial from PySide import QtCore from twisted.python import log -from twisted.internet import threads +from twisted.internet import threads, reactor from twisted.internet.defer import CancelledError +from leap.bitmask.logs.utils import get_logger from leap.common.check import leap_assert, leap_assert_type -logger = logging.getLogger(__name__) +logger = get_logger() class AbstractBootstrapper(QtCore.QObject): @@ -156,7 +155,8 @@ class AbstractBootstrapper(QtCore.QObject): data = {self.PASSED_KEY: True, self.ERROR_KEY: ""} if isinstance(signal, basestring): if self._signaler is not None: - self._signaler.signal(signal, data) + reactor.callFromThread( + self._signaler.signal, signal, data) else: logger.warning("Tried to notify but no signaler found") else: diff --git a/src/leap/bitmask/services/eip/conductor.py b/src/leap/bitmask/services/eip/conductor.py index 3fc88724..3386dddf 100644 --- a/src/leap/bitmask/services/eip/conductor.py +++ b/src/leap/bitmask/services/eip/conductor.py @@ -20,10 +20,9 @@ EIP Conductor module. This handles Qt Signals and triggers the calls to the backend, where the VPNProcess has been initialized. """ -import logging - from PySide import QtCore +from leap.bitmask.logs.utils import get_logger from leap.bitmask.gui import statemachines from leap.bitmask.services import EIP_SERVICE from leap.bitmask.services import get_service_display_name @@ -31,7 +30,7 @@ from leap.bitmask.services.eip.connection import EIPConnection from leap.bitmask.platform_init import IS_MAC QtDelayedCall = QtCore.QTimer.singleShot -logger = logging.getLogger(__name__) +logger = get_logger() class EIPConductor(object): diff --git a/src/leap/bitmask/services/eip/darwinvpnlauncher.py b/src/leap/bitmask/services/eip/darwinvpnlauncher.py index f83e0170..17fc11c2 100644 --- a/src/leap/bitmask/services/eip/darwinvpnlauncher.py +++ b/src/leap/bitmask/services/eip/darwinvpnlauncher.py @@ -19,15 +19,15 @@ Darwin VPN launcher implementation. """ import commands import getpass -import logging import os import sys +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services.eip.vpnlauncher import VPNLauncher from leap.bitmask.services.eip.vpnlauncher import VPNLauncherException from leap.bitmask.util import get_path_prefix -logger = logging.getLogger(__name__) +logger = get_logger() class EIPNoTunKextLoaded(VPNLauncherException): diff --git a/src/leap/bitmask/services/eip/eipbootstrapper.py b/src/leap/bitmask/services/eip/eipbootstrapper.py index f78113bc..7a331d71 100644 --- a/src/leap/bitmask/services/eip/eipbootstrapper.py +++ b/src/leap/bitmask/services/eip/eipbootstrapper.py @@ -17,11 +17,11 @@ """ EIP bootstrapping """ -import logging import os from leap.bitmask.config.providerconfig import ProviderConfig from leap.bitmask.crypto.certs import download_client_cert +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services import download_service_config from leap.bitmask.services.abstractbootstrapper import AbstractBootstrapper from leap.bitmask.services.eip.eipconfig import EIPConfig @@ -29,7 +29,7 @@ from leap.common import certs as leap_certs from leap.common.check import leap_assert, leap_assert_type from leap.common.files import check_and_fix_urw_only -logger = logging.getLogger(__name__) +logger = get_logger() class EIPBootstrapper(AbstractBootstrapper): diff --git a/src/leap/bitmask/services/eip/eipconfig.py b/src/leap/bitmask/services/eip/eipconfig.py index f4d6b216..43328af9 100644 --- a/src/leap/bitmask/services/eip/eipconfig.py +++ b/src/leap/bitmask/services/eip/eipconfig.py @@ -26,12 +26,13 @@ import ipaddr from leap.bitmask.config import flags from leap.bitmask.config.providerconfig import ProviderConfig +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services import ServiceConfig from leap.bitmask.services.eip.eipspec import get_schema from leap.bitmask.util import get_path_prefix from leap.common.check import leap_assert, leap_assert_type -logger = logging.getLogger(__name__) +logger = get_logger() def get_eipconfig_path(domain, relative=True): @@ -160,12 +161,17 @@ class VPNGatewaySelector(object): def get_gateways_country_code(self): """ Return a dict with ipaddress -> country code mapping. + Return None if there are no locations specified. - :rtype: dict + :rtype: dict or None """ country_codes = {} locations = self._eipconfig.get_locations() + + if not locations: + return + gateways = self._eipconfig.get_gateways() for idx, gateway in enumerate(gateways): @@ -302,6 +308,24 @@ class EIPConfig(ServiceConfig): logger.error("Invalid ip address in config: %s" % (ip_addr_str,)) return None + def get_gateway_ports(self, index=0): + """ + Return the ports of the gateway. + + :param index: the gateway number to get the ports from + :type index: int + + :rtype: list of int + """ + gateways = self.get_gateways() + leap_assert(len(gateways) > 0, "We don't have any gateway!") + if index > len(gateways): + index = 0 + logger.warning("Provided an unknown gateway index %s, " + + "defaulting to 0") + + return gateways[index]["capabilities"]["ports"] + def get_client_cert_path(self, providerconfig=None, about_to_download=False): diff --git a/src/leap/bitmask/services/eip/linuxvpnlauncher.py b/src/leap/bitmask/services/eip/linuxvpnlauncher.py index a3ab408b..cf14a8f9 100644 --- a/src/leap/bitmask/services/eip/linuxvpnlauncher.py +++ b/src/leap/bitmask/services/eip/linuxvpnlauncher.py @@ -18,11 +18,11 @@ Linux VPN launcher implementation. """ import commands -import logging import os import sys from leap.bitmask.config import flags +from leap.bitmask.logs.utils import get_logger from leap.bitmask.util.privilege_policies import LinuxPolicyChecker from leap.bitmask.util.privilege_policies import NoPolkitAuthAgentAvailable from leap.bitmask.util.privilege_policies import NoPkexecAvailable @@ -31,7 +31,7 @@ from leap.bitmask.services.eip.vpnlauncher import VPNLauncherException from leap.bitmask.util import get_path_prefix, force_eval from leap.bitmask.util import first -logger = logging.getLogger(__name__) +logger = get_logger() COM = commands diff --git a/src/leap/bitmask/services/eip/vpnlauncher.py b/src/leap/bitmask/services/eip/vpnlauncher.py index 72e19413..c48f857c 100644 --- a/src/leap/bitmask/services/eip/vpnlauncher.py +++ b/src/leap/bitmask/services/eip/vpnlauncher.py @@ -19,7 +19,6 @@ Platform independant VPN launcher interface. """ import getpass import hashlib -import logging import os import stat @@ -27,6 +26,7 @@ from abc import ABCMeta, abstractmethod from functools import partial from leap.bitmask.config import flags +from leap.bitmask.logs.utils import get_logger from leap.bitmask.backend.settings import Settings, GATEWAY_AUTOMATIC from leap.bitmask.config.providerconfig import ProviderConfig from leap.bitmask.platform_init import IS_LINUX @@ -35,7 +35,7 @@ from leap.bitmask.util import force_eval from leap.common.check import leap_assert, leap_assert_type -logger = logging.getLogger(__name__) +logger = get_logger() class VPNLauncherException(Exception): @@ -106,12 +106,15 @@ class VPNLauncher(object): UP_SCRIPT = None DOWN_SCRIPT = None + PREFERRED_PORTS = ("443", "80", "53", "1194") + @classmethod @abstractmethod def get_gateways(kls, eipconfig, providerconfig): """ - Return the selected gateways for a given provider, looking at the EIP - config file. + Return a list with the selected gateways for a given provider, looking + at the EIP config file. + Each item of the list is a tuple containing (gateway, port). :param eipconfig: eip configuration object :type eipconfig: EIPConfig @@ -122,21 +125,37 @@ class VPNLauncher(object): :rtype: list """ gateways = [] + settings = Settings() domain = providerconfig.get_domain() gateway_conf = settings.get_selected_gateway(domain) gateway_selector = VPNGatewaySelector(eipconfig) if gateway_conf == GATEWAY_AUTOMATIC: - gateways = gateway_selector.get_gateways() + gws = gateway_selector.get_gateways() else: - gateways = [gateway_conf] + gws = [gateway_conf] - if not gateways: + if not gws: logger.error('No gateway was found!') raise VPNLauncherException('No gateway was found!') - logger.debug("Using gateways ips: {0}".format(', '.join(gateways))) + for idx, gw in enumerate(gws): + ports = eipconfig.get_gateway_ports(idx) + + the_port = "1194" # default port + + # pick the port preferring this order: + for port in kls.PREFERRED_PORTS: + if port in ports: + the_port = port + break + else: + continue + + gateways.append((gw, the_port)) + + logger.debug("Using gateways (ip, port): {0!r}".format(gateways)) return gateways @classmethod @@ -194,8 +213,8 @@ class VPNLauncher(object): gateways = kls.get_gateways(eipconfig, providerconfig) - for gw in gateways: - args += ['--remote', gw, '1194', 'udp'] + for ip, port in gateways: + args += ['--remote', ip, port, 'udp'] args += [ '--client', diff --git a/src/leap/bitmask/services/eip/vpnprocess.py b/src/leap/bitmask/services/eip/vpnprocess.py index 8dc6021f..586b50f5 100644 --- a/src/leap/bitmask/services/eip/vpnprocess.py +++ b/src/leap/bitmask/services/eip/vpnprocess.py @@ -18,7 +18,6 @@ VPN Manager, spawned in a custom processProtocol. """ import commands -import logging import os import shutil import socket @@ -39,6 +38,7 @@ except ImportError: from leap.bitmask.config import flags from leap.bitmask.config.providerconfig import ProviderConfig +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services.eip import get_vpn_launcher from leap.bitmask.services.eip import linuxvpnlauncher from leap.bitmask.services.eip.eipconfig import EIPConfig @@ -47,13 +47,12 @@ from leap.bitmask.util import first, force_eval from leap.bitmask.platform_init import IS_MAC, IS_LINUX from leap.common.check import leap_assert, leap_assert_type -logger = logging.getLogger(__name__) -vpnlog = logging.getLogger('leap.openvpn') - from twisted.internet import defer, protocol, reactor from twisted.internet import error as internet_error from twisted.internet.task import LoopingCall +logger = get_logger() + class VPNObserver(object): """ @@ -884,7 +883,7 @@ class VPNProcess(protocol.ProcessProtocol, VPNManager): """ # truncate the newline line = data[:-1] - vpnlog.info(line) + logger.info(line) self._vpn_observer.watch(line) def processExited(self, reason): @@ -961,9 +960,11 @@ class VPNProcess(protocol.ProcessProtocol, VPNManager): :rtype: list """ - gateways = self._launcher.get_gateways( + gateways_ports = self._launcher.get_gateways( self._eipconfig, self._providerconfig) - return gateways + + # filter out ports since we don't need that info + return [gateway for gateway, port in gateways_ports] # shutdown diff --git a/src/leap/bitmask/services/eip/windowsvpnlauncher.py b/src/leap/bitmask/services/eip/windowsvpnlauncher.py index 3f1ed43b..aaa3e45f 100644 --- a/src/leap/bitmask/services/eip/windowsvpnlauncher.py +++ b/src/leap/bitmask/services/eip/windowsvpnlauncher.py @@ -17,12 +17,11 @@ """ Windows VPN launcher implementation. """ -import logging - +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services.eip.vpnlauncher import VPNLauncher from leap.common.check import leap_assert -logger = logging.getLogger(__name__) +logger = get_logger() class WindowsVPNLauncher(VPNLauncher): diff --git a/src/leap/bitmask/services/mail/conductor.py b/src/leap/bitmask/services/mail/conductor.py index 0fb9f4fa..68197d9d 100644 --- a/src/leap/bitmask/services/mail/conductor.py +++ b/src/leap/bitmask/services/mail/conductor.py @@ -17,18 +17,17 @@ """ Mail Services Conductor """ -import logging - from leap.bitmask.config import flags +from leap.bitmask.logs.utils import get_logger from leap.bitmask.gui import statemachines from leap.bitmask.services.mail import connection as mail_connection from leap.bitmask.services.mail.emailfirewall import get_email_firewall -from leap.common.events import events_pb2 as leap_events +from leap.common.events import catalog from leap.common.events import register as leap_register -logger = logging.getLogger(__name__) +logger = get_logger() class IMAPControl(object): @@ -42,15 +41,12 @@ class IMAPControl(object): self.imap_machine = None self.imap_connection = None - leap_register(signal=leap_events.IMAP_SERVICE_STARTED, - callback=self._handle_imap_events, - reqcbk=lambda req, resp: None) - leap_register(signal=leap_events.IMAP_SERVICE_FAILED_TO_START, - callback=self._handle_imap_events, - reqcbk=lambda req, resp: None) - leap_register(signal=leap_events.IMAP_CLIENT_LOGIN, - callback=self._handle_imap_events, - reqcbk=lambda req, resp: None) + leap_register(event=catalog.IMAP_SERVICE_STARTED, + callback=self._handle_imap_events) + leap_register(event=catalog.IMAP_SERVICE_FAILED_TO_START, + callback=self._handle_imap_events) + leap_register(event=catalog.IMAP_CLIENT_LOGIN, + callback=self._handle_imap_events) def set_imap_connection(self, imap_connection): """ @@ -77,25 +73,29 @@ class IMAPControl(object): self._backend.imap_stop_service() - def _handle_imap_events(self, req): + def _handle_imap_events(self, event, content): """ Callback handler for the IMAP events - :param req: Request type - :type req: leap.common.events.events_pb2.SignalRequest + :param event: The event that triggered the callback. + :type event: str + :param content: The content of the event. + :type content: list """ - if req.event == leap_events.IMAP_SERVICE_STARTED: + if event == catalog.IMAP_SERVICE_STARTED: self._on_imap_connected() - elif req.event == leap_events.IMAP_SERVICE_FAILED_TO_START: + elif event == catalog.IMAP_SERVICE_FAILED_TO_START: self._on_imap_failed() - elif req.event == leap_events.IMAP_CLIENT_LOGIN: + elif event == catalog.IMAP_CLIENT_LOGIN: self._on_mail_client_logged_in() def _on_mail_client_logged_in(self): """ On mail client logged in, fetch incoming mail. """ - self._controller.imap_service_fetch() + # XXX needs to be adapted to the new-ish incoming mail service. + # Doing nothing for now, this could be moved to mail package itself. + logger.debug("A MUA has logged in, should react by forcing a fetch.") def _on_imap_connecting(self): """ @@ -124,12 +124,10 @@ class SMTPControl(object): self.smtp_connection = None self.smtp_machine = None - leap_register(signal=leap_events.SMTP_SERVICE_STARTED, - callback=self._handle_smtp_events, - reqcbk=lambda req, resp: None) - leap_register(signal=leap_events.SMTP_SERVICE_FAILED_TO_START, - callback=self._handle_smtp_events, - reqcbk=lambda req, resp: None) + leap_register(event=catalog.SMTP_SERVICE_STARTED, + callback=self._handle_smtp_events) + leap_register(event=catalog.SMTP_SERVICE_FAILED_TO_START, + callback=self._handle_smtp_events) def set_smtp_connection(self, smtp_connection): """ @@ -158,16 +156,18 @@ class SMTPControl(object): self.smtp_connection.qtsigs.disconnecting_signal.emit() self._backend.smtp_stop_service() - def _handle_smtp_events(self, req): + def _handle_smtp_events(self, event, content): """ Callback handler for the SMTP events. - :param req: Request type - :type req: leap.common.events.events_pb2.SignalRequest + :param event: The event that triggered the callback. + :type event: str + :param content: The content of the event. + :type content: list """ - if req.event == leap_events.SMTP_SERVICE_STARTED: + if event == catalog.SMTP_SERVICE_STARTED: self.on_smtp_connected() - elif req.event == leap_events.SMTP_SERVICE_FAILED_TO_START: + elif event == catalog.SMTP_SERVICE_FAILED_TO_START: self.on_smtp_failed() def on_smtp_connecting(self): @@ -262,7 +262,7 @@ class MailConductor(IMAPControl, SMTPControl): if self._firewall is not None: self._firewall.start() if not offline: - logger.debug("not starting smtp in offline mode") + logger.debug("Starting smtp service...") self.start_smtp_service(download_if_needed=download_if_needed) self.start_imap_service() diff --git a/src/leap/bitmask/services/mail/imap.py b/src/leap/bitmask/services/mail/imap.py index 5db18cb9..5934756d 100644 --- a/src/leap/bitmask/services/mail/imap.py +++ b/src/leap/bitmask/services/mail/imap.py @@ -17,14 +17,16 @@ """ Initialization of imap service """ -import logging import os import sys +from leap.bitmask.logs.utils import get_logger +from leap.mail.constants import INBOX_NAME from leap.mail.imap.service import imap +from leap.mail.incoming.service import IncomingMail, INCOMING_CHECK_PERIOD from twisted.python import log -logger = logging.getLogger(__name__) +logger = get_logger() # The name of the environment variable that has to be # set to override the default time value, in seconds. @@ -49,6 +51,9 @@ def get_mail_check_period(): logger.warning("Unhandled error while getting %s: %r" % ( INCOMING_CHECK_PERIOD_ENV, exc)) + + if period is None: + period = INCOMING_CHECK_PERIOD return period @@ -61,12 +66,34 @@ def start_imap_service(*args, **kwargs): from leap.bitmask.config import flags logger.debug('Launching imap service') - override_period = get_mail_check_period() - if override_period: - kwargs['check_period'] = override_period - if flags.MAIL_LOGFILE: log.startLogging(open(flags.MAIL_LOGFILE, 'w')) log.startLogging(sys.stdout) return imap.run_service(*args, **kwargs) + + +def start_incoming_mail_service(keymanager, soledad, imap_factory, userid): + """ + Initalizes and starts the incomming mail service. + + :returns: a Deferred that will be fired with the IncomingMail instance + """ + def setUpIncomingMail(inbox): + incoming_mail = IncomingMail( + keymanager, + soledad, + inbox.collection, + userid, + check_period=get_mail_check_period()) + return incoming_mail + + # XXX: do I really need to know here how to get a mailbox?? + # XXX: ideally, the parent service in mail would take care of initializing + # the account, and passing the mailbox to the incoming service. + # In an even better world, we just would subscribe to a channel that would + # pass us the serialized object to be inserted. + acc = imap_factory.theAccount + d = acc.callWhenReady(lambda _: acc.getMailbox(INBOX_NAME)) + d.addCallback(setUpIncomingMail) + return d diff --git a/src/leap/bitmask/services/mail/imapcontroller.py b/src/leap/bitmask/services/mail/imapcontroller.py index d0bf4c34..e5313477 100644 --- a/src/leap/bitmask/services/mail/imapcontroller.py +++ b/src/leap/bitmask/services/mail/imapcontroller.py @@ -17,12 +17,10 @@ """ IMAP service controller. """ -import logging - +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services.mail import imap - -logger = logging.getLogger(__name__) +logger = get_logger() class IMAPController(object): @@ -43,9 +41,12 @@ class IMAPController(object): self._soledad = soledad self._keymanager = keymanager - self.imap_service = None + # XXX: this should live in its own controller + # or, better, just be managed by a composite Mail Service in + # leap.mail. self.imap_port = None self.imap_factory = None + self.incoming_mail_service = None def start_imap_service(self, userid, offline=False): """ @@ -58,46 +59,53 @@ class IMAPController(object): """ logger.debug('Starting imap service') - self.imap_service, self.imap_port, \ - self.imap_factory = imap.start_imap_service( - self._soledad, - self._keymanager, - userid=userid, - offline=offline) + self.imap_port, self.imap_factory = imap.start_imap_service( + self._soledad, + userid=userid) + + def start_incoming_service(incoming_mail): + d = incoming_mail.startService() + d.addCallback(lambda started: incoming_mail) + return d + + def assign_incoming_service(incoming_mail): + self.incoming_mail_service = incoming_mail + return incoming_mail if offline is False: - logger.debug("Starting loop") - self.imap_service.start_loop() + d = imap.start_incoming_mail_service( + self._keymanager, + self._soledad, + self.imap_factory, + userid) + d.addCallback(start_incoming_service) + d.addCallback(assign_incoming_service) + d.addErrback(lambda f: logger.error(f.printTraceback())) - def stop_imap_service(self, cv): + def stop_imap_service(self): """ Stop IMAP service (fetcher, factory and port). - - :param cv: A condition variable to which we can signal when imap - indeed stops. - :type cv: threading.Condition """ - if self.imap_service is not None: + if self.incoming_mail_service is not None: # Stop the loop call in the fetcher - self.imap_service.stop() - self.imap_service = None + # XXX BUG -- the deletion of the reference should be made + # after stopService() triggers its deferred (ie, cleanup has been + # made) + self.incoming_mail_service.stopService() + self.incoming_mail_service = None + + if self.imap_port is not None: # Stop listening on the IMAP port self.imap_port.stopListening() # Stop the protocol - self.imap_factory.theAccount.closed = True - self.imap_factory.doStop(cv) - else: - # Release the condition variable so the caller doesn't have to wait - cv.acquire() - cv.notify() - cv.release() + self.imap_factory.doStop() def fetch_incoming_mail(self): """ Fetch incoming mail. """ - if self.imap_service: + if self.incoming_mail_service is not None: logger.debug('Client connected, fetching mail...') - self.imap_service.fetch() + self.incoming_mail_service.fetch() diff --git a/src/leap/bitmask/services/mail/plumber.py b/src/leap/bitmask/services/mail/plumber.py index 1af65c5d..43203f0c 100644 --- a/src/leap/bitmask/services/mail/plumber.py +++ b/src/leap/bitmask/services/mail/plumber.py @@ -17,6 +17,8 @@ """ Utils for manipulating local mailboxes. """ +# TODO --- this module has not yet catched up with 0.9.0 + import getpass import logging import os @@ -28,17 +30,15 @@ from twisted.internet import defer from leap.bitmask.backend.settings import Settings from leap.bitmask.config.providerconfig import ProviderConfig +from leap.bitmask.logs.utils import get_logger from leap.bitmask.provider import get_provider_path from leap.bitmask.services.soledad.soledadbootstrapper import get_db_paths from leap.bitmask.util import flatten, get_path_prefix -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.soledadstore import SoledadStore +from leap.mail.imap.account import IMAPAccount from leap.soledad.client import Soledad -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger = get_logger() def initialize_soledad(uuid, email, passwd, @@ -140,11 +140,7 @@ class MBOXPlumber(object): self.sol = initialize_soledad( self.uuid, self.userid, self.passwd, secrets, localdb, "/tmp", "/tmp") - memstore = MemoryStore( - permanent_store=SoledadStore(self.sol), - write_period=5) - self.acct = SoledadBackedAccount(self.userid, self.sol, - memstore=memstore) + self.acct = IMAPAccount(self.userid, self.sol) return True # diff --git a/src/leap/bitmask/services/mail/smtpbootstrapper.py b/src/leap/bitmask/services/mail/smtpbootstrapper.py index 9dd61488..cd871803 100644 --- a/src/leap/bitmask/services/mail/smtpbootstrapper.py +++ b/src/leap/bitmask/services/mail/smtpbootstrapper.py @@ -17,11 +17,11 @@ """ SMTP bootstrapping """ -import logging import os from leap.bitmask.config.providerconfig import ProviderConfig from leap.bitmask.crypto.certs import download_client_cert +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services import download_service_config from leap.bitmask.services.abstractbootstrapper import AbstractBootstrapper from leap.bitmask.services.mail.smtpconfig import SMTPConfig @@ -31,7 +31,7 @@ from leap.common import certs as leap_certs from leap.common.check import leap_assert from leap.common.files import check_and_fix_urw_only -logger = logging.getLogger(__name__) +logger = get_logger() class NoSMTPHosts(Exception): @@ -120,6 +120,7 @@ class SMTPBootstrapper(AbstractBootstrapper): self._provider_config, about_to_download=True) from leap.mail.smtp import setup_smtp_gateway + self._smtp_service, self._smtp_port = setup_smtp_gateway( port=2013, userid=self._userid, @@ -152,7 +153,7 @@ class SMTPBootstrapper(AbstractBootstrapper): self._provider_config = ProviderConfig.get_provider_config(domain) self._keymanager = keymanager self._smtp_config = SMTPConfig() - self._userid = userid + self._userid = str(userid) self._download_if_needed = download_if_needed try: diff --git a/src/leap/bitmask/services/mail/smtpconfig.py b/src/leap/bitmask/services/mail/smtpconfig.py index 09f90314..2d8de411 100644 --- a/src/leap/bitmask/services/mail/smtpconfig.py +++ b/src/leap/bitmask/services/mail/smtpconfig.py @@ -17,16 +17,16 @@ """ SMTP configuration """ -import logging import os from leap.bitmask.config.providerconfig import ProviderConfig +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services import ServiceConfig from leap.bitmask.services.mail.smtpspec import get_schema from leap.bitmask.util import get_path_prefix from leap.common.check import leap_assert, leap_assert_type -logger = logging.getLogger(__name__) +logger = get_logger() class SMTPConfig(ServiceConfig): diff --git a/src/leap/bitmask/services/soledad/soledadbootstrapper.py b/src/leap/bitmask/services/soledad/soledadbootstrapper.py index 2044a27c..57ae3849 100644 --- a/src/leap/bitmask/services/soledad/soledadbootstrapper.py +++ b/src/leap/bitmask/services/soledad/soledadbootstrapper.py @@ -17,37 +17,37 @@ """ Soledad bootstrapping """ -import logging import os import socket import sys -import time -from ssl import SSLError from sqlite3 import ProgrammingError as sqlite_ProgrammingError from u1db import errors as u1db_errors -from twisted.internet import threads +from twisted.internet import defer, reactor from zope.proxy import sameProxiedObjects from pysqlcipher.dbapi2 import ProgrammingError as sqlcipher_ProgrammingError from leap.bitmask.config import flags from leap.bitmask.config.providerconfig import ProviderConfig from leap.bitmask.crypto.srpauth import SRPAuth +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services import download_service_config from leap.bitmask.services.abstractbootstrapper import AbstractBootstrapper from leap.bitmask.services.soledad.soledadconfig import SoledadConfig from leap.bitmask.util import first, is_file, is_empty_file, make_address from leap.bitmask.util import get_path_prefix -from leap.bitmask.platform_init import IS_WIN +from leap.bitmask.util import here +from leap.bitmask.platform_init import IS_WIN, IS_MAC from leap.common.check import leap_assert, leap_assert_type, leap_check from leap.common.files import which from leap.keymanager import KeyManager, openpgp from leap.keymanager.errors import KeyNotFound from leap.soledad.common.errors import InvalidAuthTokenError -from leap.soledad.client import Soledad, BootstrapSequenceError +from leap.soledad.client import Soledad +from leap.soledad.client.secrets import BootstrapSequenceError -logger = logging.getLogger(__name__) +logger = get_logger() """ These mocks are replicated from imap tests and the repair utility. @@ -59,34 +59,6 @@ soledad client, and a switch to remote sync-able mode during runtime. """ -class Mock(object): - """ - A generic simple mock class - """ - def __init__(self, return_value=None): - self._return = return_value - - def __call__(self, *args, **kwargs): - return self._return - - -class MockSharedDB(object): - """ - Mocked SharedDB object to replace in soledad before - instantiating it in offline mode. - """ - get_doc = Mock() - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - -# TODO these exceptions could be moved to soledad itself -# after settling this down. - - class SoledadSyncError(Exception): message = "Error while syncing Soledad" @@ -132,10 +104,6 @@ class SoledadBootstrapper(AbstractBootstrapper): PUBKEY_KEY = "user[public_key]" MAX_INIT_RETRIES = 10 - MAX_SYNC_RETRIES = 10 - WAIT_MAX_SECONDS = 600 - # WAIT_STEP_SECONDS = 1 - WAIT_STEP_SECONDS = 5 def __init__(self, signaler=None): AbstractBootstrapper.__init__(self, signaler) @@ -145,32 +113,35 @@ class SoledadBootstrapper(AbstractBootstrapper): self._provider_config = None self._soledad_config = None - self._keymanager = None self._download_if_needed = False self._user = "" - self._password = "" + self._password = u"" self._address = "" self._uuid = "" self._srpauth = None + self._soledad = None + self._keymanager = None @property - def keymanager(self): - return self._keymanager + def srpauth(self): + if flags.OFFLINE is True: + return None + if self._srpauth is None: + leap_assert(self._provider_config is not None, + "We need a provider config") + self._srpauth = SRPAuth(self._provider_config) + return self._srpauth @property def soledad(self): return self._soledad @property - def srpauth(self): - if flags.OFFLINE is True: - return None - leap_assert(self._provider_config is not None, - "We need a provider config") - return SRPAuth(self._provider_config) + def keymanager(self): + return self._keymanager # initialization @@ -188,14 +159,19 @@ class SoledadBootstrapper(AbstractBootstrapper): self._address = username self._password = password self._uuid = uuid - try: - self.load_and_sync_soledad(uuid, offline=True) - self._signaler.signal(self._signaler.soledad_offline_finished) - except Exception as e: + + def error(failure): # TODO: we should handle more specific exceptions in here - logger.exception(e) + logger.exception(failure.value) self._signaler.signal(self._signaler.soledad_offline_failed) + d = self.load_and_sync_soledad(uuid, offline=True) + d.addCallback( + lambda _: self._signaler.signal( + self._signaler.soledad_offline_finished)) + d.addErrback(error) + return d + def _get_soledad_local_params(self, uuid, offline=False): """ Return the locals parameters needed for the soledad initialization. @@ -229,25 +205,19 @@ class SoledadBootstrapper(AbstractBootstrapper): :return: server_url, cert_file :rtype: tuple """ - if uuid is None: - uuid = self.srpauth.get_uuid() - if offline is True: server_url = "http://localhost:9999/" cert_file = "" else: + if uuid is None: + uuid = self.srpauth.get_uuid() server_url = self._pick_server(uuid) cert_file = self._provider_config.get_ca_cert_path() return server_url, cert_file - def _soledad_sync_errback(self, failure): - failure.trap(InvalidAuthTokenError) - # in the case of an invalid token we have already turned off mail and - # warned the user in _do_soledad_sync() - def _do_soledad_init(self, uuid, secrets_path, local_db_path, - server_url, cert_file, token): + server_url, cert_file, token, syncable): """ Initialize soledad, retry if necessary and raise an exception if we can't succeed. @@ -273,7 +243,7 @@ class SoledadBootstrapper(AbstractBootstrapper): logger.debug("Trying to init soledad....") self._try_soledad_init( uuid, secrets_path, local_db_path, - server_url, cert_file, token) + server_url, cert_file, token, syncable) logger.debug("Soledad has been initialized.") return except Exception as exc: @@ -286,15 +256,19 @@ class SoledadBootstrapper(AbstractBootstrapper): logger.exception(exc) raise SoledadInitError() - def load_and_sync_soledad(self, uuid=None, offline=False): + def load_and_sync_soledad(self, uuid=u"", offline=False): """ Once everthing is in the right place, we instantiate and sync Soledad :param uuid: the uuid of the user, used in offline mode. - :type uuid: unicode, or None. + :type uuid: unicode. :param offline: whether to instantiate soledad for offline use. :type offline: bool + + :return: A Deferred which fires when soledad is sync, or which fails + with SoledadInitError or SoledadSyncError + :rtype: defer.Deferred """ local_param = self._get_soledad_local_params(uuid, offline) remote_param = self._get_soledad_server_params(uuid, offline) @@ -302,34 +276,50 @@ class SoledadBootstrapper(AbstractBootstrapper): secrets_path, local_db_path, token = local_param server_url, cert_file = remote_param + if offline: + return self._load_soledad_nosync( + uuid, secrets_path, local_db_path, cert_file, token) + + else: + return self._load_soledad_online(uuid, secrets_path, local_db_path, + server_url, cert_file, token) + + def _load_soledad_online(self, uuid, secrets_path, local_db_path, + server_url, cert_file, token): + syncable = True try: self._do_soledad_init(uuid, secrets_path, local_db_path, - server_url, cert_file, token) - except SoledadInitError: + server_url, cert_file, token, syncable) + except SoledadInitError as e: # re-raise the exceptions from try_init, # we're currently handling the retries from the # soledad-launcher in the gui. - raise + return defer.fail(e) leap_assert(not sameProxiedObjects(self._soledad, None), "Null soledad, error while initializing") - if flags.OFFLINE: - self._init_keymanager(self._address, token) - else: - try: - address = make_address( - self._user, self._provider_config.get_domain()) - self._init_keymanager(address, token) - self._keymanager.get_key( - address, openpgp.OpenPGPKey, - private=True, fetch_remote=False) - d = threads.deferToThread(self._do_soledad_sync) - d.addErrback(self._soledad_sync_errback) - except KeyNotFound: - logger.debug("Key not found. Generating key for %s" % - (address,)) - self._do_soledad_sync() + address = make_address( + self._user, self._provider_config.get_domain()) + syncer = Syncer(self._soledad, self._signaler) + + d = self._init_keymanager(address, token) + d.addCallback(lambda _: syncer.sync()) + d.addErrback(self._soledad_sync_errback) + return d + + def _load_soledad_nosync(self, uuid, secrets_path, local_db_path, + cert_file, token): + syncable = False + self._do_soledad_init(uuid, secrets_path, local_db_path, + "", cert_file, token, syncable) + d = self._init_keymanager(self._address, token) + return d + + def _soledad_sync_errback(self, failure): + failure.trap(InvalidAuthTokenError) + # in the case of an invalid token we have already turned off mail and + # warned the user def _pick_server(self, uuid): """ @@ -355,54 +345,8 @@ class SoledadBootstrapper(AbstractBootstrapper): logger.debug("Using soledad server url: %s" % (server_url,)) return server_url - def _do_soledad_sync(self): - """ - Do several retries to get an initial soledad sync. - """ - # and now, let's sync - sync_tries = self.MAX_SYNC_RETRIES - step = self.WAIT_STEP_SECONDS - max_wait = self.WAIT_MAX_SECONDS - while sync_tries > 0: - wait = 0 - try: - logger.debug("Trying to sync soledad....") - self._try_soledad_sync() - while self.soledad.syncing: - time.sleep(step) - wait += step - if wait >= max_wait: - raise SoledadSyncError("timeout!") - logger.debug("Soledad has been synced!") - # so long, and thanks for all the fish - return - except SoledadSyncError: - # maybe it's my connection, but I'm getting - # ssl handshake timeouts and read errors quite often. - # A particularly big sync is a disaster. - # This deserves further investigation, maybe the - # retry strategy can be pushed to u1db, or at least - # it's something worthy to talk about with the - # ubuntu folks. - sync_tries += 1 - msg = "Sync failed, retrying... (retry {0} of {1})".format( - sync_tries, self.MAX_SYNC_RETRIES) - logger.warning(msg) - continue - except InvalidAuthTokenError: - self._signaler.signal( - self._signaler.soledad_invalid_auth_token) - raise - except Exception as e: - # XXX release syncing lock - logger.exception("Unhandled error while syncing " - "soledad: %r" % (e,)) - break - - raise SoledadSyncError() - def _try_soledad_init(self, uuid, secrets_path, local_db_path, - server_url, cert_file, auth_token): + server_url, cert_file, auth_token, syncable): """ Try to initialize soledad. @@ -425,9 +369,6 @@ class SoledadBootstrapper(AbstractBootstrapper): # (issue #3309) encoding = sys.getfilesystemencoding() - # XXX We should get a flag in soledad itself - if flags.OFFLINE is True: - Soledad._shared_db = MockSharedDB() try: self._soledad = Soledad( uuid, @@ -437,7 +378,8 @@ class SoledadBootstrapper(AbstractBootstrapper): server_url=server_url, cert_file=cert_file.encode(encoding), auth_token=auth_token, - defer_encryption=True) + defer_encryption=True, + syncable=syncable) # XXX All these errors should be handled by soledad itself, # and return a subclass of SoledadInitializationFailed @@ -456,34 +398,6 @@ class SoledadBootstrapper(AbstractBootstrapper): "Soledad: %r" % (exc,)) raise - def _try_soledad_sync(self): - """ - Try to sync soledad. - Raises SoledadSyncError if not successful. - """ - try: - logger.debug("BOOTSTRAPPER: trying to sync Soledad....") - # pass defer_decryption=False to get inline decryption - # for debugging. - self._soledad.sync(defer_decryption=True) - except SSLError as exc: - logger.error("%r" % (exc,)) - raise SoledadSyncError("Failed to sync soledad") - except u1db_errors.InvalidGeneration as exc: - logger.error("%r" % (exc,)) - raise SoledadSyncError("u1db: InvalidGeneration") - except (sqlite_ProgrammingError, sqlcipher_ProgrammingError) as e: - logger.exception("%r" % (e,)) - raise - except InvalidAuthTokenError: - # token is invalid, probably expired - logger.error('Invalid auth token while trying to sync Soledad') - raise - except Exception as exc: - logger.exception("Unhandled error while syncing " - "soledad: %r" % (exc,)) - raise SoledadSyncError("Failed to sync soledad") - def _download_config(self): """ Download the Soledad config for the given provider @@ -526,6 +440,9 @@ class SoledadBootstrapper(AbstractBootstrapper): except IndexError as e: logger.debug("Couldn't find the gpg binary!") logger.exception(e) + if IS_MAC: + gpgbin = os.path.abspath( + os.path.join(here(), "apps", "mail", "gpg")) leap_check(gpgbin is not None, "Could not find gpg binary") return gpgbin @@ -537,12 +454,12 @@ class SoledadBootstrapper(AbstractBootstrapper): :type address: str :param token: the auth token for accessing webapp. :type token: str + :rtype: Deferred """ - srp_auth = self.srpauth logger.debug('initializing keymanager...') - if flags.OFFLINE is True: - args = (address, "https://localhost", self._soledad) + if flags.OFFLINE: + nickserver_uri = "https://localhost" kwargs = { "ca_cert_path": "", "api_uri": "", @@ -551,45 +468,44 @@ class SoledadBootstrapper(AbstractBootstrapper): "gpgbinary": self._get_gpg_bin_path() } else: - args = ( - address, - "https://nicknym.%s:6425" % ( - self._provider_config.get_domain(),), - self._soledad - ) + nickserver_uri = "https://nicknym.%s:6425" % ( + self._provider_config.get_domain(),) kwargs = { "token": token, "ca_cert_path": self._provider_config.get_ca_cert_path(), "api_uri": self._provider_config.get_api_uri(), "api_version": self._provider_config.get_api_version(), - "uid": srp_auth.get_uuid(), + "uid": self.srpauth.get_uuid(), "gpgbinary": self._get_gpg_bin_path() } - try: - self._keymanager = KeyManager(*args, **kwargs) - except KeyNotFound: - logger.debug('key for %s not found.' % address) - except Exception as exc: - logger.exception(exc) - raise + self._keymanager = KeyManager(address, nickserver_uri, self._soledad, + **kwargs) if flags.OFFLINE is False: # make sure key is in server logger.debug('Trying to send key to server...') - try: - self._keymanager.send_key(openpgp.OpenPGPKey) - except KeyNotFound: - logger.debug('No key found for %s, will generate soon.' - % address) - except Exception as exc: - logger.error("Error sending key to server.") - logger.exception(exc) - # but we do not raise + + def send_errback(failure): + if failure.check(KeyNotFound): + logger.debug( + 'No key found for %s, it might be because soledad not ' + 'synced yet or it will generate it soon.' % address) + else: + logger.error("Error sending key to server.") + logger.exception(failure.value) + # but we do not raise + + d = self._keymanager.send_key(openpgp.OpenPGPKey) + d.addErrback(send_errback) + return d + else: + return defer.succeed(None) def _gen_key(self): """ Generates the key pair if needed, uploads it to the webapp and nickserver + :rtype: Deferred """ leap_assert(self._provider_config is not None, "We need a provider configuration!") @@ -600,30 +516,31 @@ class SoledadBootstrapper(AbstractBootstrapper): self._user, self._provider_config.get_domain()) logger.debug("Retrieving key for %s" % (address,)) - try: - self._keymanager.get_key( - address, openpgp.OpenPGPKey, private=True, fetch_remote=False) - return - except KeyNotFound: - logger.debug("Key not found. Generating key for %s" % (address,)) - - # generate key - try: - self._keymanager.gen_key(openpgp.OpenPGPKey) - except Exception as exc: - logger.error("Error while generating key!") - logger.exception(exc) - raise - - # send key - try: - self._keymanager.send_key(openpgp.OpenPGPKey) - except Exception as exc: - logger.error("Error while sending key!") - logger.exception(exc) - raise - - logger.debug("Key generated successfully.") + def if_not_found_generate(failure): + failure.trap(KeyNotFound) + logger.debug("Key not found. Generating key for %s" + % (address,)) + d = self._keymanager.gen_key(openpgp.OpenPGPKey) + d.addCallbacks(send_key, log_key_error("generating")) + return d + + def send_key(_): + d = self._keymanager.send_key(openpgp.OpenPGPKey) + d.addCallbacks( + lambda _: logger.debug("Key generated successfully."), + log_key_error("sending")) + + def log_key_error(step): + def log_err(failure): + logger.error("Error while %s key!", (step,)) + logger.exception(failure.value) + return failure + return log_err + + d = self._keymanager.get_key( + address, openpgp.OpenPGPKey, private=True, fetch_remote=False) + d.addErrback(if_not_found_generate) + return d def run_soledad_setup_checks(self, provider_config, user, password, download_if_needed=False): @@ -640,6 +557,8 @@ class SoledadBootstrapper(AbstractBootstrapper): files if the have changed since the time it was previously downloaded. :type download_if_needed: bool + + :return: Deferred """ leap_assert_type(provider_config, ProviderConfig) @@ -651,25 +570,103 @@ class SoledadBootstrapper(AbstractBootstrapper): if flags.OFFLINE: signal_finished = self._signaler.soledad_offline_finished - signal_failed = self._signaler.soledad_offline_failed - else: - signal_finished = self._signaler.soledad_bootstrap_finished - signal_failed = self._signaler.soledad_bootstrap_failed + self._signaler.signal(signal_finished) + return defer.succeed(True) + + signal_finished = self._signaler.soledad_bootstrap_finished + signal_failed = self._signaler.soledad_bootstrap_failed try: + # XXX FIXME make this async too! (use txrequests) + # Also, why the fuck would we want to download it *every time*? + # We should be fine by using last-time config, or at least + # trying it. self._download_config() - - # soledad config is ok, let's proceed to load and sync soledad uuid = self.srpauth.get_uuid() - self.load_and_sync_soledad(uuid) - - if not flags.OFFLINE: - self._gen_key() - - self._signaler.signal(signal_finished) except Exception as e: # TODO: we should handle more specific exceptions in here self._soledad = None self._keymanager = None - logger.exception("Error while bootstrapping Soledad: %r" % (e, )) + logger.exception("Error while bootstrapping Soledad: %r" % (e,)) self._signaler.signal(signal_failed) + return defer.succeed(None) + + # soledad config is ok, let's proceed to load and sync soledad + d = self.load_and_sync_soledad(uuid) + d.addCallback(lambda _: self._gen_key()) + d.addCallback(lambda _: self._signaler.signal(signal_finished)) + return d + + +class Syncer(object): + """ + Takes care of retries, timeouts and other issues while syncing + """ + # XXX: the timeout and proably all the stuff here should be moved to + # soledad + + MAX_SYNC_RETRIES = 10 + WAIT_MAX_SECONDS = 600 + + def __init__(self, soledad, signaler): + self._tries = 0 + self._soledad = soledad + self._signaler = signaler + + def sync(self): + self._callback_deferred = defer.Deferred() + self._try_sync() + return self._callback_deferred + + def _try_sync(self): + logger.debug("BOOTSTRAPPER: trying to sync Soledad....") + # pass defer_decryption=False to get inline decryption + # for debugging. + self._sync_deferred = self._soledad.sync(defer_decryption=True) + self._sync_deferred.addCallbacks(self._success, self._error) + self._timeout_delayed_call = reactor.callLater(self.WAIT_MAX_SECONDS, + self._timeout) + + def _success(self, result): + logger.debug("Soledad has been synced!") + self._timeout_delayed_call.cancel() + self._callback_deferred.callback(result) + # so long, and thanks for all the fish + + def _error(self, failure): + self._timeout_delayed_call.cancel() + if failure.check(InvalidAuthTokenError): + logger.error('Invalid auth token while trying to sync Soledad') + self._signaler.signal( + self._signaler.soledad_invalid_auth_token) + self._callback_deferred.fail(failure) + elif failure.check(sqlite_ProgrammingError, + sqlcipher_ProgrammingError): + logger.exception("%r" % (failure.value,)) + self._callback_deferred.fail(failure) + else: + logger.error("%r" % (failure.value,)) + self._retry() + + def _timeout(self): + # maybe it's my connection, but I'm getting + # ssl handshake timeouts and read errors quite often. + # A particularly big sync is a disaster. + # This deserves further investigation, maybe the + # retry strategy can be pushed to u1db, or at least + # it's something worthy to talk about with the + # ubuntu folks. + self._sync_deferred.cancel() + self._retry() + + def _retry(self): + self._tries += 1 + if self._tries < self.MAX_SYNC_RETRIES: + msg = "Sync failed, retrying... (retry {0} of {1})".format( + self._tries, self.MAX_SYNC_RETRIES) + logger.warning(msg) + self._try_sync() + else: + logger.error("Sync failed {0} times".format(self._tries)) + self._callback_deferred.errback( + SoledadSyncError("Too many retries")) diff --git a/src/leap/bitmask/services/soledad/soledadconfig.py b/src/leap/bitmask/services/soledad/soledadconfig.py index d3cc7da4..8052bcdb 100644 --- a/src/leap/bitmask/services/soledad/soledadconfig.py +++ b/src/leap/bitmask/services/soledad/soledadconfig.py @@ -17,12 +17,11 @@ """ Soledad configuration """ -import logging - +from leap.bitmask.logs.utils import get_logger from leap.bitmask.services import ServiceConfig from leap.bitmask.services.soledad.soledadspec import get_schema -logger = logging.getLogger(__name__) +logger = get_logger() class SoledadConfig(ServiceConfig): |