From 82586b4717851e882bdd0d378191595bf9ef0535 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 12 Jan 2014 21:22:51 -0400 Subject: make plumber use the cached uuid instead of authenticating --- src/leap/bitmask/services/mail/repair.py | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) (limited to 'src/leap/bitmask/services') diff --git a/src/leap/bitmask/services/mail/repair.py b/src/leap/bitmask/services/mail/repair.py index 660e9f11..e921e207 100644 --- a/src/leap/bitmask/services/mail/repair.py +++ b/src/leap/bitmask/services/mail/repair.py @@ -23,8 +23,8 @@ import os from collections import defaultdict +from leap.bitmask.config.leapsettings import LeapSettings from leap.bitmask.config.providerconfig import ProviderConfig -from leap.bitmask.crypto.srpauth import SRPAuth from leap.bitmask.util import get_path_prefix from leap.bitmask.services.soledad.soledadbootstrapper import get_db_paths @@ -104,6 +104,8 @@ class MBOXPlumber(object): user, provider = userid.split('@') self.user = user self.sol = None + self._settings = LeapSettings() + provider_config_path = os.path.join( get_path_prefix(), "leap", "providers", @@ -114,34 +116,17 @@ class MBOXPlumber(object): print "could not load provider config!" return self.exit() - self.srp = SRPAuth(provider_config) - self.srp.authentication_finished.connect(self.repair_account) - - def start_auth(self): - """ - returns the user identifier for a given provider. - - :param provider: the provider to which we authenticate against. - """ - print "Authenticating with provider..." - self.d = self.srp.authenticate(self.user, self.passwd) - def repair_account(self, *args): """ Gets the user id for this account. """ - print "Got authenticated." - - # XXX this won't be needed anymore after we keep a local - # cache of user uuids, so we'll be able to pick it from - # there. - self.uuid = self.srp.get_uuid() + self.uuid = self._settings.get_uuid(self.userid) if not self.uuid: - print "Got BAD UUID from provider!" + print "Cannot get UUID from settings. Log in at least once." return self.exit() print "UUID: %s" % (self.uuid) - secrets, localdb = get_db_paths(self.uid) + secrets, localdb = get_db_paths(self.uuid) self.sol = initialize_soledad( self.uuid, self.userid, self.passwd, @@ -168,7 +153,7 @@ class MBOXPlumber(object): print "There are %s messages" % (len_mbox,) last_ok = True if mbox.last_uid == len_mbox else False - uids_iter = (doc.content['uid'] for doc in mbox.messages.get_all()) + uids_iter = mbox.messages.all_msg_iter() dupes = self._has_dupes(uids_iter) if last_ok and not dupes: print "Mbox does not need repair." @@ -204,7 +189,6 @@ class MBOXPlumber(object): def exit(self): from twisted.internet import reactor - self.d.cancel() if self.sol: self.sol.close() try: @@ -224,7 +208,7 @@ def repair_account(userid): # go mario! plumber = MBOXPlumber(userid, passwd) - reactor.callLater(1, plumber.start_auth) + reactor.callLater(1, plumber.repair_account) reactor.run() -- cgit v1.2.3 From 36d634ad980bd260a3d93f8005725bc2dc3527f7 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 12 Jan 2014 21:28:21 -0400 Subject: rename repair to plumber to suit more generic functionality --- src/leap/bitmask/services/mail/plumber.py | 228 ++++++++++++++++++++++++++++++ src/leap/bitmask/services/mail/repair.py | 223 ----------------------------- 2 files changed, 228 insertions(+), 223 deletions(-) create mode 100644 src/leap/bitmask/services/mail/plumber.py delete mode 100644 src/leap/bitmask/services/mail/repair.py (limited to 'src/leap/bitmask/services') diff --git a/src/leap/bitmask/services/mail/plumber.py b/src/leap/bitmask/services/mail/plumber.py new file mode 100644 index 00000000..4ecbc361 --- /dev/null +++ b/src/leap/bitmask/services/mail/plumber.py @@ -0,0 +1,228 @@ +# -*- coding: utf-8 -*- +# plumber.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Utils for manipulating local mailboxes. +""" +import logging +import getpass +import os + +from collections import defaultdict + +from leap.bitmask.config.leapsettings import LeapSettings +from leap.bitmask.config.providerconfig import ProviderConfig +from leap.bitmask.util import get_path_prefix +from leap.bitmask.services.soledad.soledadbootstrapper import get_db_paths + +from leap.mail.imap.account import SoledadBackedAccount +from leap.soledad.client import Soledad + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def initialize_soledad(uuid, email, passwd, + secrets, localdb, + gnupg_home, tempdir): + """ + Initializes soledad by hand + + :param email: ID for the user + :param gnupg_home: path to home used by gnupg + :param tempdir: path to temporal dir + :rtype: Soledad instance + """ + # XXX TODO unify with an authoritative source of mocks + # for soledad (or partial initializations). + # This is copied from the imap tests. + + server_url = "http://provider" + cert_file = "" + + class Mock(object): + def __init__(self, return_value=None): + self._return = return_value + + def __call__(self, *args, **kwargs): + return self._return + + class MockSharedDB(object): + + get_doc = Mock() + put_doc = Mock() + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) + + def __call__(self): + return self + + Soledad._shared_db = MockSharedDB() + soledad = Soledad( + uuid, + passwd, + secrets, + localdb, + server_url, + cert_file) + + return soledad + + +class MBOXPlumber(object): + """ + An class that can fix things inside a soledadbacked account. + The idea is to gather in this helper different fixes for mailboxes + that can be invoked when data migration in the client is needed. + """ + + def __init__(self, userid, passwd): + """ + Initializes the plumber with all that's needed to authenticate + against the provider. + + :param userid: user identifier, foo@bar + :type userid: basestring + :param passwd: the soledad passphrase + :type passwd: basestring + """ + self.userid = userid + self.passwd = passwd + user, provider = userid.split('@') + self.user = user + self.sol = None + self._settings = LeapSettings() + + provider_config_path = os.path.join( + get_path_prefix(), + "leap", "providers", + provider, "provider.json") + provider_config = ProviderConfig() + loaded = provider_config.load(provider_config_path) + if not loaded: + print "could not load provider config!" + return self.exit() + + def repair_account(self, *args): + """ + Gets the user id for this account. + """ + self.uuid = self._settings.get_uuid(self.userid) + if not self.uuid: + print "Cannot get UUID from settings. Log in at least once." + return self.exit() + print "UUID: %s" % (self.uuid) + + secrets, localdb = get_db_paths(self.uuid) + + self.sol = initialize_soledad( + self.uuid, self.userid, self.passwd, + secrets, localdb, "/tmp", "/tmp") + + self.acct = SoledadBackedAccount(self.userid, self.sol) + for mbox_name in self.acct.mailboxes: + self.repair_mbox_uids(mbox_name) + print "done." + self.exit() + + def repair_mbox_uids(self, mbox_name): + """ + Repairs indexes for a given mbox + + :param mbox_name: mailbox to repair + :type mbox_name: basestring + """ + print + print "REPAIRING INDEXES FOR MAILBOX %s" % (mbox_name,) + print "----------------------------------------------" + mbox = self.acct.getMailbox(mbox_name) + len_mbox = mbox.getMessageCount() + print "There are %s messages" % (len_mbox,) + + last_ok = True if mbox.last_uid == len_mbox else False + uids_iter = mbox.messages.all_msg_iter() + dupes = self._has_dupes(uids_iter) + if last_ok and not dupes: + print "Mbox does not need repair." + return + + msgs = mbox.messages.get_all() + for zindex, doc in enumerate(msgs): + mindex = zindex + 1 + old_uid = doc.content['uid'] + doc.content['uid'] = mindex + self.sol.put_doc(doc) + if mindex != old_uid: + print "%s -> %s (%s)" % (mindex, doc.content['uid'], old_uid) + + old_last_uid = mbox.last_uid + mbox.last_uid = len_mbox + print "LAST UID: %s (%s)" % (mbox.last_uid, old_last_uid) + + def _has_dupes(self, sequence): + """ + Returns True if the given sequence of ints has duplicates. + + :param sequence: a sequence of ints + :type sequence: sequence + :rtype: bool + """ + d = defaultdict(lambda: 0) + for uid in sequence: + d[uid] += 1 + if d[uid] != 1: + return True + return False + + def exit(self): + from twisted.internet import reactor + if self.sol: + self.sol.close() + try: + reactor.stop() + except Exception: + pass + return + + +def repair_account(userid): + """ + Starts repair process for a given account. + :param userid: the user id (email-like) + """ + from twisted.internet import reactor + passwd = unicode(getpass.getpass("Passphrase: ")) + + # go mario! + plumber = MBOXPlumber(userid, passwd) + reactor.callLater(1, plumber.repair_account) + reactor.run() + + +if __name__ == "__main__": + import sys + + logging.basicConfig() + + if len(sys.argv) != 3: + print "Usage: plumber [repair|import] " + sys.exit(1) + + # this would be better with a dict if it grows + if sys.argv[1] == "repair": + repair_account(sys.argv[2]) + if sys.argv[1] == "import": + print "Not implemented yet." diff --git a/src/leap/bitmask/services/mail/repair.py b/src/leap/bitmask/services/mail/repair.py deleted file mode 100644 index e921e207..00000000 --- a/src/leap/bitmask/services/mail/repair.py +++ /dev/null @@ -1,223 +0,0 @@ -# -*- coding: utf-8 -*- -# repair.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Utils for repairing mailbox indexes. -""" -import logging -import getpass -import os - -from collections import defaultdict - -from leap.bitmask.config.leapsettings import LeapSettings -from leap.bitmask.config.providerconfig import ProviderConfig -from leap.bitmask.util import get_path_prefix -from leap.bitmask.services.soledad.soledadbootstrapper import get_db_paths - -from leap.mail.imap.account import SoledadBackedAccount -from leap.soledad.client import Soledad - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -def initialize_soledad(uuid, email, passwd, - secrets, localdb, - gnupg_home, tempdir): - """ - Initializes soledad by hand - - :param email: ID for the user - :param gnupg_home: path to home used by gnupg - :param tempdir: path to temporal dir - :rtype: Soledad instance - """ - # XXX TODO unify with an authoritative source of mocks - # for soledad (or partial initializations). - # This is copied from the imap tests. - - server_url = "http://provider" - cert_file = "" - - class Mock(object): - def __init__(self, return_value=None): - self._return = return_value - - def __call__(self, *args, **kwargs): - return self._return - - class MockSharedDB(object): - - get_doc = Mock() - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() - soledad = Soledad( - uuid, - passwd, - secrets, - localdb, - server_url, - cert_file) - - return soledad - - -class MBOXPlumber(object): - """ - An class that can fix things inside a soledadbacked account. - The idea is to gather in this helper different fixes for mailboxes - that can be invoked when data migration in the client is needed. - """ - - def __init__(self, userid, passwd): - """ - Initializes the plumber with all that's needed to authenticate - against the provider. - - :param userid: user identifier, foo@bar - :type userid: basestring - :param passwd: the soledad passphrase - :type passwd: basestring - """ - self.userid = userid - self.passwd = passwd - user, provider = userid.split('@') - self.user = user - self.sol = None - self._settings = LeapSettings() - - provider_config_path = os.path.join( - get_path_prefix(), - "leap", "providers", - provider, "provider.json") - provider_config = ProviderConfig() - loaded = provider_config.load(provider_config_path) - if not loaded: - print "could not load provider config!" - return self.exit() - - def repair_account(self, *args): - """ - Gets the user id for this account. - """ - self.uuid = self._settings.get_uuid(self.userid) - if not self.uuid: - print "Cannot get UUID from settings. Log in at least once." - return self.exit() - print "UUID: %s" % (self.uuid) - - secrets, localdb = get_db_paths(self.uuid) - - self.sol = initialize_soledad( - self.uuid, self.userid, self.passwd, - secrets, localdb, "/tmp", "/tmp") - - self.acct = SoledadBackedAccount(self.userid, self.sol) - for mbox_name in self.acct.mailboxes: - self.repair_mbox_uids(mbox_name) - print "done." - self.exit() - - def repair_mbox_uids(self, mbox_name): - """ - Repairs indexes for a given mbox - - :param mbox_name: mailbox to repair - :type mbox_name: basestring - """ - print - print "REPAIRING INDEXES FOR MAILBOX %s" % (mbox_name,) - print "----------------------------------------------" - mbox = self.acct.getMailbox(mbox_name) - len_mbox = mbox.getMessageCount() - print "There are %s messages" % (len_mbox,) - - last_ok = True if mbox.last_uid == len_mbox else False - uids_iter = mbox.messages.all_msg_iter() - dupes = self._has_dupes(uids_iter) - if last_ok and not dupes: - print "Mbox does not need repair." - return - - msgs = mbox.messages.get_all() - for zindex, doc in enumerate(msgs): - mindex = zindex + 1 - old_uid = doc.content['uid'] - doc.content['uid'] = mindex - self.sol.put_doc(doc) - if mindex != old_uid: - print "%s -> %s (%s)" % (mindex, doc.content['uid'], old_uid) - - old_last_uid = mbox.last_uid - mbox.last_uid = len_mbox - print "LAST UID: %s (%s)" % (mbox.last_uid, old_last_uid) - - def _has_dupes(self, sequence): - """ - Returns True if the given sequence of ints has duplicates. - - :param sequence: a sequence of ints - :type sequence: sequence - :rtype: bool - """ - d = defaultdict(lambda: 0) - for uid in sequence: - d[uid] += 1 - if d[uid] != 1: - return True - return False - - def exit(self): - from twisted.internet import reactor - if self.sol: - self.sol.close() - try: - reactor.stop() - except Exception: - pass - return - - -def repair_account(userid): - """ - Starts repair process for a given account. - :param userid: the user id (email-like) - """ - from twisted.internet import reactor - passwd = unicode(getpass.getpass("Passphrase: ")) - - # go mario! - plumber = MBOXPlumber(userid, passwd) - reactor.callLater(1, plumber.repair_account) - reactor.run() - - -if __name__ == "__main__": - import sys - - logging.basicConfig() - - if len(sys.argv) != 2: - print "Usage: repair " - sys.exit(1) - repair_account(sys.argv[1]) -- cgit v1.2.3 From 7f9fa030ed44a7db6ced5b359c49dadc0a781b8a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 12 Jan 2014 22:56:31 -0400 Subject: able to import maildir --- src/leap/bitmask/services/mail/plumber.py | 114 +++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 9 deletions(-) (limited to 'src/leap/bitmask/services') diff --git a/src/leap/bitmask/services/mail/plumber.py b/src/leap/bitmask/services/mail/plumber.py index 4ecbc361..49514655 100644 --- a/src/leap/bitmask/services/mail/plumber.py +++ b/src/leap/bitmask/services/mail/plumber.py @@ -22,10 +22,13 @@ import getpass import os from collections import defaultdict +from functools import partial + +from twisted.internet import defer from leap.bitmask.config.leapsettings import LeapSettings from leap.bitmask.config.providerconfig import ProviderConfig -from leap.bitmask.util import get_path_prefix +from leap.bitmask.util import flatten, get_path_prefix from leap.bitmask.services.soledad.soledadbootstrapper import get_db_paths from leap.mail.imap.account import SoledadBackedAccount @@ -89,20 +92,23 @@ class MBOXPlumber(object): that can be invoked when data migration in the client is needed. """ - def __init__(self, userid, passwd): + def __init__(self, userid, passwd, mdir=None): """ - Initializes the plumber with all that's needed to authenticate + Initialize the plumber with all that's needed to authenticate against the provider. :param userid: user identifier, foo@bar :type userid: basestring :param passwd: the soledad passphrase :type passwd: basestring + :param mdir: a path to a maildir to import + :type mdir: str or None """ self.userid = userid self.passwd = passwd user, provider = userid.split('@') self.user = user + self.mdir = mdir self.sol = None self._settings = LeapSettings() @@ -116,9 +122,9 @@ class MBOXPlumber(object): print "could not load provider config!" return self.exit() - def repair_account(self, *args): + def _init_local_soledad(self): """ - Gets the user id for this account. + Initialize local Soledad instance. """ self.uuid = self._settings.get_uuid(self.userid) if not self.uuid: @@ -131,8 +137,16 @@ class MBOXPlumber(object): self.sol = initialize_soledad( self.uuid, self.userid, self.passwd, secrets, localdb, "/tmp", "/tmp") - self.acct = SoledadBackedAccount(self.userid, self.sol) + # + # Account repairing + # + + def repair_account(self, *args): + """ + Repair mbox uids for all mboxes in this account. + """ + self._init_local_soledad() for mbox_name in self.acct.mailboxes: self.repair_mbox_uids(mbox_name) print "done." @@ -140,7 +154,7 @@ class MBOXPlumber(object): def repair_mbox_uids(self, mbox_name): """ - Repairs indexes for a given mbox + Repair indexes for a given mbox. :param mbox_name: mailbox to repair :type mbox_name: basestring @@ -159,6 +173,7 @@ class MBOXPlumber(object): print "Mbox does not need repair." return + # XXX CHANGE? ---- msgs = mbox.messages.get_all() for zindex, doc in enumerate(msgs): mindex = zindex + 1 @@ -174,7 +189,7 @@ class MBOXPlumber(object): def _has_dupes(self, sequence): """ - Returns True if the given sequence of ints has duplicates. + Return True if the given sequence of ints has duplicates. :param sequence: a sequence of ints :type sequence: sequence @@ -187,6 +202,71 @@ class MBOXPlumber(object): return True return False + # + # Maildir import + # + def import_mail(self, mail_filename): + """ + Import a single mail into a mailbox. + + :param mbox: the Mailbox instance to save in. + :type mbox: SoledadMailbox + :param mail_filename: the filename to the mail file to save + :type mail_filename: basestring + :return: a deferred + """ + def saved(_): + print "message added" + + with open(mail_filename) as f: + mail_string = f.read() + uid = self._mbox.getUIDNext() + print "saving with UID: %s" % uid + d = self._mbox.messages.add_msg(mail_string, uid=uid) + return d + + def import_maildir(self, mbox_name="INBOX"): + """ + Import all mails in a maildir. + + We will process all subfolders as beloging + to the same mailbox (cur, new, tmp). + """ + # TODO parse hierarchical subfolders into + # inferior mailboxes. + + if not os.path.isdir(self.mdir): + print "ERROR: maildir path does not exist." + return + + self._init_local_soledad() + mbox = self.acct.getMailbox(mbox_name) + self._mbox = mbox + len_mbox = mbox.getMessageCount() + + mail_files_g = flatten( + map(partial(os.path.join, f), files) + for f, _, files in os.walk(self.mdir)) + + # we only coerce the generator to give the + # len, but we could skip than and inform at the end. + mail_files = list(mail_files_g) + print "Got %s mails to import into %s (%s)" % ( + len(mail_files), mbox_name, len_mbox) + + def all_saved(_): + print "all messages imported" + + deferreds = [] + for f_name in mail_files: + deferreds.append(self.import_mail(f_name)) + d1 = defer.gatherResults(deferreds, consumeErrors=False) + d1.addCallback(all_saved) + d1.addCallback(self._cbExit) + + def _cbExit(self, ignored): + return self.exit() + def exit(self): from twisted.internet import reactor if self.sol: @@ -200,7 +280,8 @@ class MBOXPlumber(object): def repair_account(userid): """ - Starts repair process for a given account. + Start repair process for a given account. + :param userid: the user id (email-like) """ from twisted.internet import reactor @@ -212,6 +293,21 @@ def repair_account(userid): reactor.run() +def import_maildir(userid, maildir_path): + """ + Start import-maildir process for a given account. + + :param userid: the user id (email-like) + """ + from twisted.internet import reactor + passwd = unicode(getpass.getpass("Passphrase: ")) + + # go mario! + plumber = MBOXPlumber(userid, passwd, mdir=maildir_path) + reactor.callLater(1, plumber.import_maildir) + reactor.run() + + if __name__ == "__main__": import sys -- cgit v1.2.3