diff options
23 files changed, 664 insertions, 436 deletions
@@ -119,5 +119,17 @@ install_wheel: # if it's the first time, you'll need to get_wheels first pip install --pre --use-wheel --no-index --find-links=../wheelhouse -r pkg/requirements.pip +gather_deps: + pipdeptree | pkg/scripts/filter-bitmask-deps + +all_leap_develop: + cd ../keymanager && git checkout develop + cd ../leap_common && git checkout develop + cd ../leap_mail && git checkout develop + cd ../soledad && git checkout develop + +all_leap_release_tags: + pkg/scripts/checkout_leap_versions.sh + clean : $(RM) $(COMPILED_UI) $(COMPILED_RESOURCES) $(COMPILED_UI:.py=.pyc) $(COMPILED_RESOURCES:.py=.pyc) diff --git a/changes/VERSION_COMPAT b/changes/VERSION_COMPAT index 1eadcbe0..ddb63835 100644 --- a/changes/VERSION_COMPAT +++ b/changes/VERSION_COMPAT @@ -4,8 +4,12 @@ # Add your changes here so we can properly update # requirements.pip during the release process. # (leave header when resetting) +# +# When bumping it during the release cycle +# remember to update also pkg/leap_versions.txt ################################################# # # BEGIN DEPENDENCY LIST ------------------------- # leap.foo.bar>=x.y.z leap.keymanager>=0.4.0 +leap.mail>=0.4.0 diff --git a/changes/bug-6654_use-correct-provider b/changes/bug-6654_use-correct-provider new file mode 100644 index 00000000..511e2fea --- /dev/null +++ b/changes/bug-6654_use-correct-provider @@ -0,0 +1 @@ +- #6654: Regression fix, login attempt is made against previously selected provider. diff --git a/changes/feature_async-api b/changes/feature_async-api new file mode 100644 index 00000000..50694f58 --- /dev/null +++ b/changes/feature_async-api @@ -0,0 +1,3 @@ +- Add support to the new async-api of soledad +- Add support to the new async-api of keymanager (Closes: 6368) +- Refactor soledad sync to do it the twisted way (Closes: #6536, #6691, #6658) diff --git a/changes/feature_sumo-tarball b/changes/feature_sumo-tarball new file mode 100644 index 00000000..c4da0aa5 --- /dev/null +++ b/changes/feature_sumo-tarball @@ -0,0 +1 @@ +- Bug #6683: Add ability to generate sumo tarball. diff --git a/pkg/leap_versions.txt b/pkg/leap_versions.txt new file mode 100644 index 00000000..b95d1b29 --- /dev/null +++ b/pkg/leap_versions.txt @@ -0,0 +1,4 @@ +soledad 0.6.3 +keymanager 0.3.8 +leap_common 0.3.9 +leap_mail 0.3.11 diff --git a/pkg/requirements-dev.pip b/pkg/requirements-dev.pip index 799376d2..45f5fa70 100644 --- a/pkg/requirements-dev.pip +++ b/pkg/requirements-dev.pip @@ -1,3 +1,13 @@ +# ------------------------------------ +# -- useful tools that you probably -- +# -- will want during development -- +# ------------------------------------ + +wheel +sphinx +ipdb +pipdeptree + # --------------------------- # -- external requirements -- # -- during development -- @@ -11,11 +21,7 @@ # to install it. (do it after python setup.py develop and it # will only install this) # -wheel -sphinx -ipdb # in case you want to install a package from a git source, you can use this: # Useful to test pre-release branches together. #-e git+https://github.com/leapcode/leap_pycommon.git@develop#egg=leap.common -#-e git+https://github.com/leapcode/soledad.git@develop#egg=leap.soledad diff --git a/pkg/scripts/checkout_leap_versions.sh b/pkg/scripts/checkout_leap_versions.sh new file mode 100755 index 00000000..9d6c6fab --- /dev/null +++ b/pkg/scripts/checkout_leap_versions.sh @@ -0,0 +1,7 @@ +#!/bin/sh +cat pkg/leap_versions.txt | while read line +do + package=$(echo $line | cut -f1 -d' ') + tag=$(echo $line | cut -f2 -d' ') + cd ../$package && git checkout $tag +done diff --git a/pkg/scripts/filter-bitmask-deps b/pkg/scripts/filter-bitmask-deps new file mode 100755 index 00000000..9808d394 --- /dev/null +++ b/pkg/scripts/filter-bitmask-deps @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +""" +Filter bitmask dependencies. + +Usage: pipdeptree | filter-bitmask-deps +""" +import fileinput + +TARGET = "leap.bitmask" + + +def get_bitmask_deps(dep_lines): + res = [] + begin = False + for dep in dep_lines: + if dep.startswith(TARGET): + begin = True + res.append(dep) + continue + elif dep.startswith(' ') and begin: + res.append(dep) + continue + if begin: + return res + + +if __name__ == "__main__": + lines = [] + for line in fileinput.input(): + lines.append(line) + + bitmask_deps = get_bitmask_deps(lines) + for line in bitmask_deps: + print line[:-1] @@ -99,20 +99,7 @@ class freeze_debianver(Command): To be used after merging the development branch onto the debian one. """ user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - proceed = str(raw_input( - "This will overwrite the file _version.py. Continue? [y/N] ")) - if proceed != "y": - print("He. You scared. Aborting.") - return - template = r""" + template = r""" # This file was generated by the `freeze_debianver` command in setup.py # Using 'versioneer.py' (0.7+) from # revision-control system data, or from the parent directory name of an @@ -122,18 +109,43 @@ class freeze_debianver(Command): version_version = '{version}' version_full = '{version_full}' """ - templatefun = r""" + templatefun = r""" def get_versions(default={}, verbose=False): return {'version': version_version, 'full': version_full} """ - subst_template = template.format( + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + proceed = str(raw_input( + "This will overwrite the file _version.py. Continue? [y/N] ")) + if proceed != "y": + print("He. You scared. Aborting.") + return + subst_template = self.template.format( version=VERSION_SHORT, - version_full=VERSION_FULL) + templatefun + version_full=VERSION_FULL) + self.templatefun with open(versioneer.versionfile_source, 'w') as f: f.write(subst_template) +def freeze_pkg_ver(path, version_short, version_full): + """ + Freeze the _version in other modules, used during the gathering of + all the leap modules in the sumo tarball. + """ + subst_template = freeze_debianver.template.format( + version=version_short, + version_full=version_full) + freeze_debianver.templatefun + with open(path, 'w') as f: + f.write(subst_template) + + cmdclass["freeze_debianver"] = freeze_debianver parsed_reqs = utils.parse_requirements() @@ -240,13 +252,169 @@ class cmd_build(versioneer_build): class cmd_sdist(versioneer_sdist): + + user_options = versioneer_sdist.user_options + \ + [('sumo', 's', + "create a 'sumo' sdist which includes the contents of all " + "the leap.* packages") + ] + boolean_options = ['sumo'] + leap_sumo_packages = ['soledad.common', 'soledad.client', + 'keymanager', 'mail', 'common'] + + def initialize_options(self): + versioneer_sdist.initialize_options(self) + self.sumo = False + def run(self): return versioneer_sdist.run(self) def make_release_tree(self, base_dir, files): versioneer_sdist.make_release_tree(self, base_dir, files) + # We need to copy the requirements to the specified path + # so that the client has a copy to do the startup checks. copy_reqs(base_dir, withsrc=True) + def make_distribution(self): + # add our extra files to the list just before building the + # tarball/zipfile. We override make_distribution() instead of run() + # because setuptools.command.sdist.run() does not lend itself to + # easy/robust subclassing (the code we need to add goes right smack + # in the middle of a 12-line method). If this were the distutils + # version, we'd override get_file_list(). + + if self.sumo: + # If '--sumo' was specified, include all the leap.* in the sdist. + vdict = _get_leap_versions() + vdict['soledad.common'] = vdict['soledad'] + vdict['soledad.client'] = vdict['soledad'] + import importlib + for module in self.leap_sumo_packages: + full_module = "leap." + module + importlib.import_module(full_module) + src_path = "src/leap/" + _fix_namespace(module) + imported_module = importlib.sys.modules[full_module] + copy_recursively( + imported_module.__path__[0] + "/", + src_path) + all_module_files = list_recursively(src_path) + self.filelist.extend(all_module_files) + freeze_pkg_ver( + src_path + "/_version.py", vdict[module], "sumo") + with open('src/leap/__init__.py', 'w') as nuke_top_init: + nuke_top_init.write('') + + # In addition, we want the tarball/zipfile to have -SUMO in the + # name, and the unpacked directory to have -SUMO too. The easiest + # way to do this is to patch self.distribution and override the + # get_fullname() method. (an alternative is to modify + # self.distribution.metadata.version, but that also affects the + # contents of PKG-INFO). + fullname = self.distribution.get_fullname() + + def get_fullname(): + return fullname + "-SUMO" + self.distribution.get_fullname = get_fullname + + try: + old_mask = os.umask(int("022", 8)) + return versioneer_sdist.make_distribution(self) + finally: + os.umask(old_mask) + for module in self.leap_sumo_packages: + # check, just in case... + if module and module != "bitmask": + shutil.rmtree("src/leap/" + _fix_namespace(module)) + + +import shutil +import glob + + +def _get_leap_versions(): + versions = {} + with open("pkg/leap_versions.txt") as vf: + lines = vf.readlines() + for line in lines: + pkg, ver = line.split('\t') + versions[pkg.strip().replace('leap_', '')] = ver.strip() + return versions + + +def _fix_namespace(path): + if path in ('soledad.common', 'soledad.client'): + return path.replace('.', '/') + return path + + +_ignore_files = ('*.pyc', '_trial*', '*.swp', '.*', 'cert', 'test*') +_ignore_dirs = ('tests', '_trial*', 'test*') +_ignore_paths = _ignore_files + _ignore_dirs + +is_excluded_path = lambda path: any( + map(lambda pattern: glob.fnmatch.fnmatch(path, pattern), + _ignore_paths)) + + +def _should_exclude(path): + folder, f = os.path.split(path) + if is_excluded_path(f): + return True + upper, leaf = os.path.split(folder) + if is_excluded_path(leaf): + return True + return False + + +def list_recursively(root_dir): + file_list = [] + for root, sub_dirs, files in os.walk(root_dir): + for f in files: + is_excluded = _should_exclude(f) + if not is_excluded: + file_list.append(os.path.join(root, f)) + return file_list + + +def _mkdir_recursively(path): + sub_path = os.path.dirname(path) + if not os.path.exists(sub_path): + _mkdir_recursively(sub_path) + if not os.path.exists(path): + os.mkdir(path) + + +def copy_recursively(source_folder, destination_folder): + + if not os.path.exists(destination_folder): + _mkdir_recursively(destination_folder) + + for root, dirs, files in os.walk(source_folder): + if _should_exclude(root): + continue + for item in files: + if _should_exclude(item): + continue + src_path = os.path.join(root, item) + dst_path = os.path.join( + destination_folder, src_path.replace(source_folder, "")) + if _should_exclude(dst_path): + continue + if os.path.exists(dst_path): + if os.stat(src_path).st_mtime > os.stat(dst_path).st_mtime: + shutil.copy2(src_path, dst_path) + else: + shutil.copy2(src_path, dst_path) + for item in dirs: + if _should_exclude(item): + continue + src_path = os.path.join(root, item) + dst_path = os.path.join( + destination_folder, src_path.replace(source_folder, "")) + if _should_exclude(dst_path): + continue + if not os.path.exists(dst_path): + os.mkdir(dst_path) cmdclass["build"] = cmd_build cmdclass["sdist"] = cmd_sdist diff --git a/src/leap/bitmask/app.py b/src/leap/bitmask/app.py index 9056d2a6..72c03cb9 100644 --- a/src/leap/bitmask/app.py +++ b/src/leap/bitmask/app.py @@ -127,9 +127,7 @@ def start_app(): } flags.STANDALONE = opts.standalone - # XXX Disabled right now since it's not tested after login refactor - # flags.OFFLINE = opts.offline - flags.OFFLINE = False + flags.OFFLINE = opts.offline flags.MAIL_LOGFILE = opts.mail_log_file flags.APP_VERSION_CHECK = opts.app_version_check flags.API_VERSION_CHECK = opts.api_version_check @@ -139,16 +137,19 @@ def start_app(): flags.CA_CERT_FILE = opts.ca_cert_file replace_stdout = True - if opts.repair or opts.import_maildir: - # We don't want too much clutter on the comand mode - # this could be more generic with a Command class. - replace_stdout = False + + # XXX mail repair commands disabled for now + # if opts.repair or opts.import_maildir: + # We don't want too much clutter on the comand mode + # this could be more generic with a Command class. + # replace_stdout = False logger = create_logger(opts.debug, opts.log_file, replace_stdout) # ok, we got logging in place, we can satisfy mail plumbing requests # and show logs there. it normally will exit there if we got that path. - do_mail_plumbing(opts) + # XXX mail repair commands disabled for now + # do_mail_plumbing(opts) try: event_server.ensure_server(event_server.SERVER_PORT) diff --git a/src/leap/bitmask/backend/backend_proxy.py b/src/leap/bitmask/backend/backend_proxy.py index b2f79a70..04046d3d 100644 --- a/src/leap/bitmask/backend/backend_proxy.py +++ b/src/leap/bitmask/backend/backend_proxy.py @@ -28,6 +28,7 @@ import time import zmq from leap.bitmask.backend.api import API, STOP_REQUEST, PING_REQUEST +from leap.bitmask.backend.settings import Settings from leap.bitmask.backend.utils import generate_zmq_certificates_if_needed from leap.bitmask.backend.utils import get_backend_certificates from leap.bitmask.config import flags @@ -58,6 +59,8 @@ class BackendProxy(object): self._socket = None + self.settings = Settings() + # initialize ZMQ stuff: context = zmq.Context() logger.debug("Connecting to server...") diff --git a/src/leap/bitmask/backend/components.py b/src/leap/bitmask/backend/components.py index 4b63af84..1efcda6c 100644 --- a/src/leap/bitmask/backend/components.py +++ b/src/leap/bitmask/backend/components.py @@ -17,13 +17,16 @@ """ Backend components """ + +# TODO [ ] Get rid of all this deferToThread mess, or at least contain +# all of it into its own threadpool. + import logging import os import socket import time from functools import partial -from threading import Condition from twisted.internet import threads, defer from twisted.python import log @@ -59,9 +62,8 @@ from leap.bitmask.util.privilege_policies import LinuxPolicyChecker from leap.common import certs as leap_certs from leap.keymanager import openpgp -from leap.keymanager.errors import KeyAddressMismatch, KeyFingerprintMismatch -from leap.soledad.client import NoStorageSecret, PassphraseTooShort +from leap.soledad.client.secrets import NoStorageSecret, PassphraseTooShort logger = logging.getLogger(__name__) @@ -777,8 +779,8 @@ class Soledad(object): """ provider_config = ProviderConfig.get_provider_config(domain) if provider_config is not None: - self._soledad_defer = threads.deferToThread( - self._soledad_bootstrapper.run_soledad_setup_checks, + sb = self._soledad_bootstrapper + self._soledad_defer = sb.run_soledad_setup_checks( provider_config, username, password, download_if_needed=True) self._soledad_defer.addCallback(self._set_proxies_cb) @@ -814,8 +816,9 @@ class Soledad(object): Signaler.soledad_offline_finished Signaler.soledad_offline_failed """ - self._soledad_bootstrapper.load_offline_soledad( + d = self._soledad_bootstrapper.load_offline_soledad( username, password, uuid) + d.addCallback(self._set_proxies_cb) def cancel_bootstrap(self): """ @@ -906,52 +909,6 @@ class Keymanager(object): # NOTE: This feature is disabled right now since is dangerous return - new_key = '' - signal = None - try: - with open(filename, 'r') as keys_file: - new_key = keys_file.read() - except IOError as e: - logger.error("IOError importing key. {0!r}".format(e)) - signal = self._signaler.keymanager_import_ioerror - self._signaler.signal(signal) - return - - keymanager = self._keymanager_proxy - try: - # NOTE: parse_openpgp_ascii_key is not in keymanager anymore - # the API for that will need some thinking - public_key, private_key = keymanager.parse_openpgp_ascii_key( - new_key) - except (KeyAddressMismatch, KeyFingerprintMismatch) as e: - logger.error(repr(e)) - signal = self._signaler.keymanager_import_datamismatch - self._signaler.signal(signal) - return - - if public_key is None or private_key is None: - signal = self._signaler.keymanager_import_missingkey - self._signaler.signal(signal) - return - - current_public_key = keymanager.get_key(username, openpgp.OpenPGPKey) - if public_key.address != current_public_key.address: - logger.error("The key does not match the ID") - signal = self._signaler.keymanager_import_addressmismatch - self._signaler.signal(signal) - return - - keymanager.delete_key(self._key) - keymanager.delete_key(self._key_priv) - keymanager.put_key(public_key) - keymanager.put_key(private_key) - keymanager.send_key(openpgp.OpenPGPKey) - - logger.debug('Import ok') - signal = self._signaler.keymanager_import_ok - - self._signaler.signal(signal) - def export_keys(self, username, filename): """ Export the given username's keys to a file. @@ -963,35 +920,50 @@ class Keymanager(object): """ keymanager = self._keymanager_proxy - public_key = keymanager.get_key(username, openpgp.OpenPGPKey) - private_key = keymanager.get_key(username, openpgp.OpenPGPKey, - private=True) - try: + def export(keys): + public_key, private_key = keys + # XXX: This is blocking. We could use writeToFD, but is POSIX only + # https://twistedmatrix.com/documents/current/api/twisted.internet.fdesc.html#writeToFD with open(filename, 'w') as keys_file: keys_file.write(public_key.key_data) keys_file.write(private_key.key_data) logger.debug('Export ok') self._signaler.signal(self._signaler.keymanager_export_ok) - except IOError as e: - logger.error("IOError exporting key. {0!r}".format(e)) + + def log_error(failure): + logger.error( + "Error exporting key. {0!r}".format(failure.value)) self._signaler.signal(self._signaler.keymanager_export_error) + dpub = keymanager.get_key(username, openpgp.OpenPGPKey) + dpriv = keymanager.get_key(username, openpgp.OpenPGPKey, + private=True) + d = defer.gatherResults([dpub, dpriv]) + d.addCallback(export) + d.addErrback(log_error) + def list_keys(self): """ List all the keys stored in the local DB. """ - keys = self._keymanager_proxy.get_all_keys() - self._signaler.signal(self._signaler.keymanager_keys_list, keys) + d = self._keymanager_proxy.get_all_keys() + d.addCallback( + lambda keys: + self._signaler.signal(self._signaler.keymanager_keys_list, keys)) def get_key_details(self, username): """ List all the keys stored in the local DB. """ - public_key = self._keymanager_proxy.get_key(username, - openpgp.OpenPGPKey) - details = (public_key.key_id, public_key.fingerprint) - self._signaler.signal(self._signaler.keymanager_key_details, details) + def signal_details(public_key): + details = (public_key.key_id, public_key.fingerprint) + self._signaler.signal(self._signaler.keymanager_key_details, + details) + + d = self._keymanager_proxy.get_key(username, + openpgp.OpenPGPKey) + d.addCallback(signal_details) class Mail(object): @@ -1070,12 +1042,10 @@ class Mail(object): """ Stop imap and wait until the service is stopped to signal that is done. """ - cv = Condition() - cv.acquire() - threads.deferToThread(self._imap_controller.stop_imap_service, cv) + # FIXME just get a fucking deferred and signal as a callback, with + # timeout and cancellability + threads.deferToThread(self._imap_controller.stop_imap_service) logger.debug('Waiting for imap service to stop.') - cv.wait(self.SERVICE_STOP_TIMEOUT) - logger.debug('IMAP stopped') self._signaler.signal(self._signaler.imap_stopped) def stop_imap_service(self): diff --git a/src/leap/bitmask/backend/settings.py b/src/leap/bitmask/backend/settings.py index 5cb4c616..ed70ca6b 100644 --- a/src/leap/bitmask/backend/settings.py +++ b/src/leap/bitmask/backend/settings.py @@ -122,37 +122,36 @@ class Settings(object): self._settings.set(provider, self.GATEWAY_KEY, gateway) self._save() - def get_uuid(self, username): + def get_uuid(self, full_user_id): """ Gets the uuid for a given username. - :param username: the full user identifier in the form user@provider - :type username: basestring + :param full_user_id: the full user identifier in the form user@provider + :type full_user_id: basestring """ - leap_assert("@" in username, + leap_assert("@" in full_user_id, "Expected username in the form user@provider") - user, provider = username.split('@') + username, provider = full_user_id.split('@') + return self._get_value(provider, full_user_id, "") - return self._get_value(provider, username, "") - - def set_uuid(self, username, value): + def set_uuid(self, full_user_id, value): """ Sets the uuid for a given username. - :param username: the full user identifier in the form user@provider - :type username: str or unicode + :param full_user_id: the full user identifier in the form user@provider + :type full_user_id: str or unicode :param value: the uuid to save or None to remove it :type value: str or unicode or None """ - leap_assert("@" in username, + leap_assert("@" in full_user_id, "Expected username in the form user@provider") - user, provider = username.split('@') + user, provider = full_user_id.split('@') if value is None: - self._settings.remove_option(provider, username) + self._settings.remove_option(provider, full_user_id) else: leap_assert(len(value) > 0, "We cannot save an empty uuid") self._add_section(provider) - self._settings.set(provider, username, value) + self._settings.set(provider, full_user_id, value) self._save() diff --git a/src/leap/bitmask/config/leapsettings.py b/src/leap/bitmask/config/leapsettings.py index 13a1e99e..fd3e1592 100644 --- a/src/leap/bitmask/config/leapsettings.py +++ b/src/leap/bitmask/config/leapsettings.py @@ -353,35 +353,3 @@ class LeapSettings(object): """ leap_assert_type(skip, bool) self._settings.setValue(self.SKIPFIRSTRUN_KEY, skip) - - def get_uuid(self, username): - """ - Gets the uuid for a given username. - - :param username: the full user identifier in the form user@provider - :type username: basestring - """ - leap_assert("@" in username, - "Expected username in the form user@provider") - user, provider = username.split('@') - return self._settings.value( - self.UUIDFORUSER_KEY % (provider, user), "") - - def set_uuid(self, username, value): - """ - Sets the uuid for a given username. - - :param username: the full user identifier in the form user@provider - :type username: str or unicode - :param value: the uuid to save or None to remove it - :type value: str or unicode or None - """ - leap_assert("@" in username, - "Expected username in the form user@provider") - user, provider = username.split('@') - key = self.UUIDFORUSER_KEY % (provider, user) - if value is None: - self._settings.remove(key) - else: - leap_assert(len(value) > 0, "We cannot save an empty uuid") - self._settings.setValue(key, value) diff --git a/src/leap/bitmask/gui/mainwindow.py b/src/leap/bitmask/gui/mainwindow.py index cbf7a636..55628847 100644 --- a/src/leap/bitmask/gui/mainwindow.py +++ b/src/leap/bitmask/gui/mainwindow.py @@ -126,6 +126,7 @@ class MainWindow(QtGui.QMainWindow, SignalTracker): self._backend = self.app.backend self._leap_signaler = self.app.signaler self._settings = self.app.settings + self._backend_settings = self._backend.settings # Login Widget self._login_widget = LoginWidget(self._backend, @@ -210,10 +211,8 @@ class MainWindow(QtGui.QMainWindow, SignalTracker): self.ui.action_wizard.triggered.connect(self._show_wizard) self.ui.action_show_logs.triggered.connect(self._show_logger_window) - # XXX hide the help menu since it only shows email information and - # right now we don't have stable mail and just confuses users. - self.ui.action_help.setVisible(False) - # self.ui.action_help.triggered.connect(self._help) + self.ui.action_help.setVisible(True) + self.ui.action_help.triggered.connect(self._help) self.ui.action_create_new_account.triggered.connect( self._on_provider_changed) @@ -1192,9 +1191,14 @@ class MainWindow(QtGui.QMainWindow, SignalTracker): self._disconnect_login_wait() user = self._login_widget.get_logged_user() - domain = self._providers.get_selected_provider() - full_user_id = make_address(user, domain) - self._mail_conductor.userid = full_user_id + # XXX the widget now gives us the full user id. + # this is confusing. + # domain = self._providers.get_selected_provider() + # full_user_id = make_address(user, domain) + + # XXX the casting to str (needed by smtp gateway) should be done + # in a better place. + self._mail_conductor.userid = str(user) self._start_eip_bootstrap() self.ui.action_create_new_account.setEnabled(True) @@ -1312,10 +1316,10 @@ class MainWindow(QtGui.QMainWindow, SignalTracker): if flags.OFFLINE: full_user_id = make_address(username, provider_domain) - uuid = self._settings.get_uuid(full_user_id) + uuid = self._backend_settings.get_uuid(full_user_id) self._mail_conductor.userid = full_user_id - if uuid is None: + if not uuid: # We don't need more visibility at the moment, # this is mostly for internal use/debug for now. logger.warning("Sorry! Log-in at least one time.") diff --git a/src/leap/bitmask/services/mail/conductor.py b/src/leap/bitmask/services/mail/conductor.py index 0fb9f4fa..42bdd032 100644 --- a/src/leap/bitmask/services/mail/conductor.py +++ b/src/leap/bitmask/services/mail/conductor.py @@ -262,7 +262,7 @@ class MailConductor(IMAPControl, SMTPControl): if self._firewall is not None: self._firewall.start() if not offline: - logger.debug("not starting smtp in offline mode") + logger.debug("Starting smtp service...") self.start_smtp_service(download_if_needed=download_if_needed) self.start_imap_service() diff --git a/src/leap/bitmask/services/mail/imap.py b/src/leap/bitmask/services/mail/imap.py index 5db18cb9..da753a0b 100644 --- a/src/leap/bitmask/services/mail/imap.py +++ b/src/leap/bitmask/services/mail/imap.py @@ -21,7 +21,9 @@ import logging import os import sys +from leap.mail.constants import INBOX_NAME from leap.mail.imap.service import imap +from leap.mail.incoming.service import IncomingMail, INCOMING_CHECK_PERIOD from twisted.python import log logger = logging.getLogger(__name__) @@ -49,6 +51,9 @@ def get_mail_check_period(): logger.warning("Unhandled error while getting %s: %r" % ( INCOMING_CHECK_PERIOD_ENV, exc)) + + if period is None: + period = INCOMING_CHECK_PERIOD return period @@ -61,12 +66,39 @@ def start_imap_service(*args, **kwargs): from leap.bitmask.config import flags logger.debug('Launching imap service') - override_period = get_mail_check_period() - if override_period: - kwargs['check_period'] = override_period - if flags.MAIL_LOGFILE: log.startLogging(open(flags.MAIL_LOGFILE, 'w')) log.startLogging(sys.stdout) return imap.run_service(*args, **kwargs) + + +def start_incoming_mail_service(keymanager, soledad, imap_factory, userid): + """ + Initalizes and starts the incomming mail service. + + :returns: a Deferred that will be fired with the IncomingMail instance + """ + def setUpIncomingMail(inbox): + incoming_mail = IncomingMail( + keymanager, + soledad, + inbox.collection, + userid, + check_period=get_mail_check_period()) + return incoming_mail + + # XXX: do I really need to know here how to get a mailbox?? + # XXX: ideally, the parent service in mail would take care of initializing + # the account, and passing the mailbox to the incoming service. + # In an even better world, we just would subscribe to a channel that would + # pass us the serialized object to be inserted. + # XXX: I think we might be at risk here because the account top object + # currently just returns as many mailbox objects as it's asked for, so + # we should be careful about concurrent operations (ie, maybe just use + # this one to do inserts, or let account have an in-memory map of + # mailboxes, and just return references to them). + acc = imap_factory.theAccount + d = acc.callWhenReady(lambda _: acc.getMailbox(INBOX_NAME)) + d.addCallback(setUpIncomingMail) + return d diff --git a/src/leap/bitmask/services/mail/imapcontroller.py b/src/leap/bitmask/services/mail/imapcontroller.py index d0bf4c34..d374ac29 100644 --- a/src/leap/bitmask/services/mail/imapcontroller.py +++ b/src/leap/bitmask/services/mail/imapcontroller.py @@ -43,9 +43,12 @@ class IMAPController(object): self._soledad = soledad self._keymanager = keymanager - self.imap_service = None + # XXX: this should live in its own controller + # or, better, just be managed by a composite Mail Service in + # leap.mail. self.imap_port = None self.imap_factory = None + self.incoming_mail_service = None def start_imap_service(self, userid, offline=False): """ @@ -58,46 +61,53 @@ class IMAPController(object): """ logger.debug('Starting imap service') - self.imap_service, self.imap_port, \ - self.imap_factory = imap.start_imap_service( - self._soledad, - self._keymanager, - userid=userid, - offline=offline) + self.imap_port, self.imap_factory = imap.start_imap_service( + self._soledad, + userid=userid) + + def start_incoming_service(incoming_mail): + d = incoming_mail.startService() + d.addCallback(lambda started: incoming_mail) + return d + + def assign_incoming_service(incoming_mail): + self.incoming_mail_service = incoming_mail + return incoming_mail if offline is False: - logger.debug("Starting loop") - self.imap_service.start_loop() + d = imap.start_incoming_mail_service( + self._keymanager, + self._soledad, + self.imap_factory, + userid) + d.addCallback(start_incoming_service) + d.addCallback(assign_incoming_service) + d.addErrback(lambda f: logger.error(f.printTraceback())) - def stop_imap_service(self, cv): + def stop_imap_service(self): """ Stop IMAP service (fetcher, factory and port). - - :param cv: A condition variable to which we can signal when imap - indeed stops. - :type cv: threading.Condition """ - if self.imap_service is not None: + if self.incoming_mail_service is not None: # Stop the loop call in the fetcher - self.imap_service.stop() - self.imap_service = None + # XXX BUG -- the deletion of the reference should be made + # after stopService() triggers its deferred (ie, cleanup has been + # made) + self.incoming_mail_service.stopService() + self.incoming_mail_service = None + + if self.imap_port is not None: # Stop listening on the IMAP port self.imap_port.stopListening() # Stop the protocol - self.imap_factory.theAccount.closed = True - self.imap_factory.doStop(cv) - else: - # Release the condition variable so the caller doesn't have to wait - cv.acquire() - cv.notify() - cv.release() + self.imap_factory.doStop() def fetch_incoming_mail(self): """ Fetch incoming mail. """ - if self.imap_service: + if self.incoming_mail_service is not None: logger.debug('Client connected, fetching mail...') - self.imap_service.fetch() + self.incoming_mail_service.fetch() diff --git a/src/leap/bitmask/services/mail/plumber.py b/src/leap/bitmask/services/mail/plumber.py index 1af65c5d..58625db5 100644 --- a/src/leap/bitmask/services/mail/plumber.py +++ b/src/leap/bitmask/services/mail/plumber.py @@ -17,6 +17,8 @@ """ Utils for manipulating local mailboxes. """ +# TODO --- this module has not yet catched up with 0.9.0 + import getpass import logging import os @@ -32,9 +34,7 @@ from leap.bitmask.provider import get_provider_path from leap.bitmask.services.soledad.soledadbootstrapper import get_db_paths from leap.bitmask.util import flatten, get_path_prefix -from leap.mail.imap.account import SoledadBackedAccount -from leap.mail.imap.memorystore import MemoryStore -from leap.mail.imap.soledadstore import SoledadStore +from leap.mail.imap.account import IMAPAccount from leap.soledad.client import Soledad logger = logging.getLogger(__name__) @@ -140,11 +140,7 @@ class MBOXPlumber(object): self.sol = initialize_soledad( self.uuid, self.userid, self.passwd, secrets, localdb, "/tmp", "/tmp") - memstore = MemoryStore( - permanent_store=SoledadStore(self.sol), - write_period=5) - self.acct = SoledadBackedAccount(self.userid, self.sol, - memstore=memstore) + self.acct = IMAPAccount(self.userid, self.sol) return True # diff --git a/src/leap/bitmask/services/mail/smtpbootstrapper.py b/src/leap/bitmask/services/mail/smtpbootstrapper.py index 9dd61488..6f45469b 100644 --- a/src/leap/bitmask/services/mail/smtpbootstrapper.py +++ b/src/leap/bitmask/services/mail/smtpbootstrapper.py @@ -120,6 +120,7 @@ class SMTPBootstrapper(AbstractBootstrapper): self._provider_config, about_to_download=True) from leap.mail.smtp import setup_smtp_gateway + self._smtp_service, self._smtp_port = setup_smtp_gateway( port=2013, userid=self._userid, @@ -152,7 +153,7 @@ class SMTPBootstrapper(AbstractBootstrapper): self._provider_config = ProviderConfig.get_provider_config(domain) self._keymanager = keymanager self._smtp_config = SMTPConfig() - self._userid = userid + self._userid = str(userid) self._download_if_needed = download_if_needed try: diff --git a/src/leap/bitmask/services/soledad/soledadbootstrapper.py b/src/leap/bitmask/services/soledad/soledadbootstrapper.py index 2044a27c..879ed1d0 100644 --- a/src/leap/bitmask/services/soledad/soledadbootstrapper.py +++ b/src/leap/bitmask/services/soledad/soledadbootstrapper.py @@ -21,13 +21,12 @@ import logging import os import socket import sys -import time from ssl import SSLError from sqlite3 import ProgrammingError as sqlite_ProgrammingError from u1db import errors as u1db_errors -from twisted.internet import threads +from twisted.internet import defer, reactor from zope.proxy import sameProxiedObjects from pysqlcipher.dbapi2 import ProgrammingError as sqlcipher_ProgrammingError @@ -45,7 +44,8 @@ from leap.common.files import which from leap.keymanager import KeyManager, openpgp from leap.keymanager.errors import KeyNotFound from leap.soledad.common.errors import InvalidAuthTokenError -from leap.soledad.client import Soledad, BootstrapSequenceError +from leap.soledad.client import Soledad +from leap.soledad.client.secrets import BootstrapSequenceError logger = logging.getLogger(__name__) @@ -59,34 +59,6 @@ soledad client, and a switch to remote sync-able mode during runtime. """ -class Mock(object): - """ - A generic simple mock class - """ - def __init__(self, return_value=None): - self._return = return_value - - def __call__(self, *args, **kwargs): - return self._return - - -class MockSharedDB(object): - """ - Mocked SharedDB object to replace in soledad before - instantiating it in offline mode. - """ - get_doc = Mock() - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - -# TODO these exceptions could be moved to soledad itself -# after settling this down. - - class SoledadSyncError(Exception): message = "Error while syncing Soledad" @@ -132,10 +104,6 @@ class SoledadBootstrapper(AbstractBootstrapper): PUBKEY_KEY = "user[public_key]" MAX_INIT_RETRIES = 10 - MAX_SYNC_RETRIES = 10 - WAIT_MAX_SECONDS = 600 - # WAIT_STEP_SECONDS = 1 - WAIT_STEP_SECONDS = 5 def __init__(self, signaler=None): AbstractBootstrapper.__init__(self, signaler) @@ -145,32 +113,35 @@ class SoledadBootstrapper(AbstractBootstrapper): self._provider_config = None self._soledad_config = None - self._keymanager = None self._download_if_needed = False self._user = "" - self._password = "" + self._password = u"" self._address = "" self._uuid = "" self._srpauth = None + self._soledad = None + self._keymanager = None @property - def keymanager(self): - return self._keymanager + def srpauth(self): + if flags.OFFLINE is True: + return None + if self._srpauth is None: + leap_assert(self._provider_config is not None, + "We need a provider config") + self._srpauth = SRPAuth(self._provider_config) + return self._srpauth @property def soledad(self): return self._soledad @property - def srpauth(self): - if flags.OFFLINE is True: - return None - leap_assert(self._provider_config is not None, - "We need a provider config") - return SRPAuth(self._provider_config) + def keymanager(self): + return self._keymanager # initialization @@ -188,14 +159,19 @@ class SoledadBootstrapper(AbstractBootstrapper): self._address = username self._password = password self._uuid = uuid - try: - self.load_and_sync_soledad(uuid, offline=True) - self._signaler.signal(self._signaler.soledad_offline_finished) - except Exception as e: + + def error(failure): # TODO: we should handle more specific exceptions in here - logger.exception(e) + logger.exception(failure.value) self._signaler.signal(self._signaler.soledad_offline_failed) + d = self.load_and_sync_soledad(uuid, offline=True) + d.addCallback( + lambda _: self._signaler.signal( + self._signaler.soledad_offline_finished)) + d.addErrback(error) + return d + def _get_soledad_local_params(self, uuid, offline=False): """ Return the locals parameters needed for the soledad initialization. @@ -229,25 +205,19 @@ class SoledadBootstrapper(AbstractBootstrapper): :return: server_url, cert_file :rtype: tuple """ - if uuid is None: - uuid = self.srpauth.get_uuid() - if offline is True: server_url = "http://localhost:9999/" cert_file = "" else: + if uuid is None: + uuid = self.srpauth.get_uuid() server_url = self._pick_server(uuid) cert_file = self._provider_config.get_ca_cert_path() return server_url, cert_file - def _soledad_sync_errback(self, failure): - failure.trap(InvalidAuthTokenError) - # in the case of an invalid token we have already turned off mail and - # warned the user in _do_soledad_sync() - def _do_soledad_init(self, uuid, secrets_path, local_db_path, - server_url, cert_file, token): + server_url, cert_file, token, syncable): """ Initialize soledad, retry if necessary and raise an exception if we can't succeed. @@ -273,7 +243,7 @@ class SoledadBootstrapper(AbstractBootstrapper): logger.debug("Trying to init soledad....") self._try_soledad_init( uuid, secrets_path, local_db_path, - server_url, cert_file, token) + server_url, cert_file, token, syncable) logger.debug("Soledad has been initialized.") return except Exception as exc: @@ -286,15 +256,19 @@ class SoledadBootstrapper(AbstractBootstrapper): logger.exception(exc) raise SoledadInitError() - def load_and_sync_soledad(self, uuid=None, offline=False): + def load_and_sync_soledad(self, uuid=u"", offline=False): """ Once everthing is in the right place, we instantiate and sync Soledad :param uuid: the uuid of the user, used in offline mode. - :type uuid: unicode, or None. + :type uuid: unicode. :param offline: whether to instantiate soledad for offline use. :type offline: bool + + :return: A Deferred which fires when soledad is sync, or which fails + with SoledadInitError or SoledadSyncError + :rtype: defer.Deferred """ local_param = self._get_soledad_local_params(uuid, offline) remote_param = self._get_soledad_server_params(uuid, offline) @@ -302,34 +276,50 @@ class SoledadBootstrapper(AbstractBootstrapper): secrets_path, local_db_path, token = local_param server_url, cert_file = remote_param + if offline: + return self._load_soledad_nosync( + uuid, secrets_path, local_db_path, cert_file, token) + + else: + return self._load_soledad_online(uuid, secrets_path, local_db_path, + server_url, cert_file, token) + + def _load_soledad_online(self, uuid, secrets_path, local_db_path, + server_url, cert_file, token): + syncable = True try: self._do_soledad_init(uuid, secrets_path, local_db_path, - server_url, cert_file, token) - except SoledadInitError: + server_url, cert_file, token, syncable) + except SoledadInitError as e: # re-raise the exceptions from try_init, # we're currently handling the retries from the # soledad-launcher in the gui. - raise + return defer.fail(e) leap_assert(not sameProxiedObjects(self._soledad, None), "Null soledad, error while initializing") - if flags.OFFLINE: - self._init_keymanager(self._address, token) - else: - try: - address = make_address( - self._user, self._provider_config.get_domain()) - self._init_keymanager(address, token) - self._keymanager.get_key( - address, openpgp.OpenPGPKey, - private=True, fetch_remote=False) - d = threads.deferToThread(self._do_soledad_sync) - d.addErrback(self._soledad_sync_errback) - except KeyNotFound: - logger.debug("Key not found. Generating key for %s" % - (address,)) - self._do_soledad_sync() + address = make_address( + self._user, self._provider_config.get_domain()) + syncer = Syncer(self._soledad, self._signaler) + + d = self._init_keymanager(address, token) + d.addCallback(lambda _: syncer.sync()) + d.addErrback(self._soledad_sync_errback) + return d + + def _load_soledad_nosync(self, uuid, secrets_path, local_db_path, + cert_file, token): + syncable = False + self._do_soledad_init(uuid, secrets_path, local_db_path, + "", cert_file, token, syncable) + d = self._init_keymanager(self._address, token) + return d + + def _soledad_sync_errback(self, failure): + failure.trap(InvalidAuthTokenError) + # in the case of an invalid token we have already turned off mail and + # warned the user def _pick_server(self, uuid): """ @@ -355,54 +345,8 @@ class SoledadBootstrapper(AbstractBootstrapper): logger.debug("Using soledad server url: %s" % (server_url,)) return server_url - def _do_soledad_sync(self): - """ - Do several retries to get an initial soledad sync. - """ - # and now, let's sync - sync_tries = self.MAX_SYNC_RETRIES - step = self.WAIT_STEP_SECONDS - max_wait = self.WAIT_MAX_SECONDS - while sync_tries > 0: - wait = 0 - try: - logger.debug("Trying to sync soledad....") - self._try_soledad_sync() - while self.soledad.syncing: - time.sleep(step) - wait += step - if wait >= max_wait: - raise SoledadSyncError("timeout!") - logger.debug("Soledad has been synced!") - # so long, and thanks for all the fish - return - except SoledadSyncError: - # maybe it's my connection, but I'm getting - # ssl handshake timeouts and read errors quite often. - # A particularly big sync is a disaster. - # This deserves further investigation, maybe the - # retry strategy can be pushed to u1db, or at least - # it's something worthy to talk about with the - # ubuntu folks. - sync_tries += 1 - msg = "Sync failed, retrying... (retry {0} of {1})".format( - sync_tries, self.MAX_SYNC_RETRIES) - logger.warning(msg) - continue - except InvalidAuthTokenError: - self._signaler.signal( - self._signaler.soledad_invalid_auth_token) - raise - except Exception as e: - # XXX release syncing lock - logger.exception("Unhandled error while syncing " - "soledad: %r" % (e,)) - break - - raise SoledadSyncError() - def _try_soledad_init(self, uuid, secrets_path, local_db_path, - server_url, cert_file, auth_token): + server_url, cert_file, auth_token, syncable): """ Try to initialize soledad. @@ -425,9 +369,6 @@ class SoledadBootstrapper(AbstractBootstrapper): # (issue #3309) encoding = sys.getfilesystemencoding() - # XXX We should get a flag in soledad itself - if flags.OFFLINE is True: - Soledad._shared_db = MockSharedDB() try: self._soledad = Soledad( uuid, @@ -437,7 +378,8 @@ class SoledadBootstrapper(AbstractBootstrapper): server_url=server_url, cert_file=cert_file.encode(encoding), auth_token=auth_token, - defer_encryption=True) + defer_encryption=True, + syncable=syncable) # XXX All these errors should be handled by soledad itself, # and return a subclass of SoledadInitializationFailed @@ -456,34 +398,6 @@ class SoledadBootstrapper(AbstractBootstrapper): "Soledad: %r" % (exc,)) raise - def _try_soledad_sync(self): - """ - Try to sync soledad. - Raises SoledadSyncError if not successful. - """ - try: - logger.debug("BOOTSTRAPPER: trying to sync Soledad....") - # pass defer_decryption=False to get inline decryption - # for debugging. - self._soledad.sync(defer_decryption=True) - except SSLError as exc: - logger.error("%r" % (exc,)) - raise SoledadSyncError("Failed to sync soledad") - except u1db_errors.InvalidGeneration as exc: - logger.error("%r" % (exc,)) - raise SoledadSyncError("u1db: InvalidGeneration") - except (sqlite_ProgrammingError, sqlcipher_ProgrammingError) as e: - logger.exception("%r" % (e,)) - raise - except InvalidAuthTokenError: - # token is invalid, probably expired - logger.error('Invalid auth token while trying to sync Soledad') - raise - except Exception as exc: - logger.exception("Unhandled error while syncing " - "soledad: %r" % (exc,)) - raise SoledadSyncError("Failed to sync soledad") - def _download_config(self): """ Download the Soledad config for the given provider @@ -537,12 +451,12 @@ class SoledadBootstrapper(AbstractBootstrapper): :type address: str :param token: the auth token for accessing webapp. :type token: str + :rtype: Deferred """ - srp_auth = self.srpauth logger.debug('initializing keymanager...') - if flags.OFFLINE is True: - args = (address, "https://localhost", self._soledad) + if flags.OFFLINE: + nickserver_uri = "https://localhost" kwargs = { "ca_cert_path": "", "api_uri": "", @@ -551,45 +465,44 @@ class SoledadBootstrapper(AbstractBootstrapper): "gpgbinary": self._get_gpg_bin_path() } else: - args = ( - address, - "https://nicknym.%s:6425" % ( - self._provider_config.get_domain(),), - self._soledad - ) + nickserver_uri = "https://nicknym.%s:6425" % ( + self._provider_config.get_domain(),) kwargs = { "token": token, "ca_cert_path": self._provider_config.get_ca_cert_path(), "api_uri": self._provider_config.get_api_uri(), "api_version": self._provider_config.get_api_version(), - "uid": srp_auth.get_uuid(), + "uid": self.srpauth.get_uuid(), "gpgbinary": self._get_gpg_bin_path() } - try: - self._keymanager = KeyManager(*args, **kwargs) - except KeyNotFound: - logger.debug('key for %s not found.' % address) - except Exception as exc: - logger.exception(exc) - raise + self._keymanager = KeyManager(address, nickserver_uri, self._soledad, + **kwargs) if flags.OFFLINE is False: # make sure key is in server logger.debug('Trying to send key to server...') - try: - self._keymanager.send_key(openpgp.OpenPGPKey) - except KeyNotFound: - logger.debug('No key found for %s, will generate soon.' - % address) - except Exception as exc: - logger.error("Error sending key to server.") - logger.exception(exc) - # but we do not raise + + def send_errback(failure): + if failure.check(KeyNotFound): + logger.debug( + 'No key found for %s, it might be because soledad not ' + 'synced yet or it will generate it soon.' % address) + else: + logger.error("Error sending key to server.") + logger.exception(failure.value) + # but we do not raise + + d = self._keymanager.send_key(openpgp.OpenPGPKey) + d.addErrback(send_errback) + return d + else: + return defer.succeed(None) def _gen_key(self): """ Generates the key pair if needed, uploads it to the webapp and nickserver + :rtype: Deferred """ leap_assert(self._provider_config is not None, "We need a provider configuration!") @@ -600,30 +513,31 @@ class SoledadBootstrapper(AbstractBootstrapper): self._user, self._provider_config.get_domain()) logger.debug("Retrieving key for %s" % (address,)) - try: - self._keymanager.get_key( - address, openpgp.OpenPGPKey, private=True, fetch_remote=False) - return - except KeyNotFound: - logger.debug("Key not found. Generating key for %s" % (address,)) - - # generate key - try: - self._keymanager.gen_key(openpgp.OpenPGPKey) - except Exception as exc: - logger.error("Error while generating key!") - logger.exception(exc) - raise - - # send key - try: - self._keymanager.send_key(openpgp.OpenPGPKey) - except Exception as exc: - logger.error("Error while sending key!") - logger.exception(exc) - raise - - logger.debug("Key generated successfully.") + def if_not_found_generate(failure): + failure.trap(KeyNotFound) + logger.debug("Key not found. Generating key for %s" + % (address,)) + d = self._keymanager.gen_key(openpgp.OpenPGPKey) + d.addCallbacks(send_key, log_key_error("generating")) + return d + + def send_key(_): + d = self._keymanager.send_key(openpgp.OpenPGPKey) + d.addCallbacks( + lambda _: logger.debug("Key generated successfully."), + log_key_error("sending")) + + def log_key_error(step): + def log_err(failure): + logger.error("Error while %s key!", (step,)) + logger.exception(failure.value) + return failure + return log_err + + d = self._keymanager.get_key( + address, openpgp.OpenPGPKey, private=True, fetch_remote=False) + d.addErrback(if_not_found_generate) + return d def run_soledad_setup_checks(self, provider_config, user, password, download_if_needed=False): @@ -640,6 +554,8 @@ class SoledadBootstrapper(AbstractBootstrapper): files if the have changed since the time it was previously downloaded. :type download_if_needed: bool + + :return: Deferred """ leap_assert_type(provider_config, ProviderConfig) @@ -651,25 +567,111 @@ class SoledadBootstrapper(AbstractBootstrapper): if flags.OFFLINE: signal_finished = self._signaler.soledad_offline_finished - signal_failed = self._signaler.soledad_offline_failed - else: - signal_finished = self._signaler.soledad_bootstrap_finished - signal_failed = self._signaler.soledad_bootstrap_failed + self._signaler.signal(signal_finished) + return defer.succeed(True) + + signal_finished = self._signaler.soledad_bootstrap_finished + signal_failed = self._signaler.soledad_bootstrap_failed try: + # XXX FIXME make this async too! (use txrequests) + # Also, why the fuck would we want to download it *every time*? + # We should be fine by using last-time config, or at least + # trying it. self._download_config() - - # soledad config is ok, let's proceed to load and sync soledad uuid = self.srpauth.get_uuid() - self.load_and_sync_soledad(uuid) - - if not flags.OFFLINE: - self._gen_key() - - self._signaler.signal(signal_finished) except Exception as e: # TODO: we should handle more specific exceptions in here self._soledad = None self._keymanager = None - logger.exception("Error while bootstrapping Soledad: %r" % (e, )) + logger.exception("Error while bootstrapping Soledad: %r" % (e,)) self._signaler.signal(signal_failed) + return defer.succeed(None) + + # soledad config is ok, let's proceed to load and sync soledad + d = self.load_and_sync_soledad(uuid) + d.addCallback(lambda _: self._gen_key()) + d.addCallback(lambda _: self._signaler.signal(signal_finished)) + return d + + +class Syncer(object): + """ + Takes care of retries, timeouts and other issues while syncing + """ + # XXX: the timeout and proably all the stuff here should be moved to + # soledad + + MAX_SYNC_RETRIES = 10 + WAIT_MAX_SECONDS = 600 + + def __init__(self, soledad, signaler): + self._tries = 0 + self._soledad = soledad + self._signaler = signaler + + def sync(self): + self._callback_deferred = defer.Deferred() + self._try_sync() + return self._callback_deferred + + def _try_sync(self): + logger.debug("BOOTSTRAPPER: trying to self Soledad....") + # pass defer_decryption=False to get inline decryption + # for debugging. + self._sync_deferred = self._soledad.sync(defer_decryption=True) + self._sync_deferred.addCallbacks(self._success, self._error) + reactor.callLater(self.WAIT_MAX_SECONDS, self._timeout) + + def _success(self, result): + logger.debug("Soledad has been synced!") + self._callback_deferred.callback(result) + # so long, and thanks for all the fish + + def _error(self, failure): + if failure.check(InvalidAuthTokenError): + logger.error('Invalid auth token while trying to self Soledad') + self._signaler.signal( + self._signaler.soledad_invalid_auth_token) + self._callback_deferred.fail(failure) + elif failure.check(sqlite_ProgrammingError, + sqlcipher_ProgrammingError): + logger.exception("%r" % (failure.value,)) + self._callback_deferred.fail(failure) + elif failure.check(SSLError): + logger.error("%r" % (failure.value,)) + self._retry() + elif failure.check(u1db_errors.InvalidGeneration): + logger.error("%r" % (failure.value,)) + self._retry() + else: + logger.exception("Unhandled error while syncing " + "soledad: %r" % (failure.value,)) + self._retry() + + def _timeout(self): + if not self._soledad.syncing: + # timeout only if is still syncing + return + + # maybe it's my connection, but I'm getting + # ssl handshake timeouts and read errors quite often. + # A particularly big sync is a disaster. + # This deserves further investigation, maybe the + # retry strategy can be pushed to u1db, or at least + # it's something worthy to talk about with the + # ubuntu folks. + self._sync_deferred.cancel() + self._retry() + + def _retry(self): + self._tries += 1 + if self._tries > self.MAX_SYNC_RETRIES: + msg = "Sync failed, retrying... (retry {0} of {1})".format( + self._tries, self.MAX_SYNC_RETRIES) + logger.warning(msg) + self._try_sync() + else: + logger.error("Sync failed {0} times".format(self._tries)) + self._callback_deferred.fail( + SoledadSyncError("Too many retries")) diff --git a/src/leap/bitmask/util/leap_argparse.py b/src/leap/bitmask/util/leap_argparse.py index 346caed5..12fd9736 100644 --- a/src/leap/bitmask/util/leap_argparse.py +++ b/src/leap/bitmask/util/leap_argparse.py @@ -74,26 +74,27 @@ def build_parser(): help='Verbosity level for openvpn logs [1-6]') # mail stuff - # XXX Disabled right now since it's not tested after login refactor - # parser.add_argument('-o', '--offline', action="store_true", - # help='Starts Bitmask in offline mode: will not ' - # 'try to sync with remote replicas for email.') - - parser.add_argument('--acct', metavar="user@provider", - nargs='?', - action="store", dest="acct", - help='Manipulate mailboxes for this account') - parser.add_argument('-r', '--repair-mailboxes', default=False, - action="store_true", dest="repair", - help='Repair mailboxes for a given account. ' - 'Use when upgrading versions after a schema ' - 'change. Use with --acct') - parser.add_argument('--import-maildir', metavar="/path/to/Maildir", - nargs='?', - action="store", dest="import_maildir", - help='Import the given maildir. Use with the ' - '--to-mbox flag to import to folders other ' - 'than INBOX. Use with --acct') + parser.add_argument('-o', '--offline', action="store_true", + help='Starts Bitmask in offline mode: will not ' + 'try to sync with remote replicas for email.') + + # XXX not yet updated to new mail api for mail 0.4.0 + + # parser.add_argument('--acct', metavar="user@provider", + #nargs='?', + #action="store", dest="acct", + #help='Manipulate mailboxes for this account') + # parser.add_argument('-r', '--repair-mailboxes', default=False, + #action="store_true", dest="repair", + #help='Repair mailboxes for a given account. ' + #'Use when upgrading versions after a schema ' + #'change. Use with --acct') + # parser.add_argument('--import-maildir', metavar="/path/to/Maildir", + #nargs='?', + #action="store", dest="import_maildir", + #help='Import the given maildir. Use with the ' + #'--to-mbox flag to import to folders other ' + #'than INBOX. Use with --acct') if not IS_RELEASE_VERSION: help_text = ("Bypasses the certificate check during provider " |