From c8dbbb6bad0463ba84ca9186693e21e9c99161a0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 26 Dec 2014 18:25:36 -0400 Subject: MessageCollections + MailboxIndexer --- src/leap/mail/mailbox_indexer.py | 254 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 src/leap/mail/mailbox_indexer.py (limited to 'src/leap/mail/mailbox_indexer.py') diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py new file mode 100644 index 0000000..bc298ea --- /dev/null +++ b/src/leap/mail/mailbox_indexer.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +# mailbox_indexer.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Local tables to store the message Unique Identifiers for a given mailbox. +""" +import re + +from leap.mail.constants import METAMSGID_RE + + +class WrongMetaDocIDError(Exception): + pass + + +class MailboxIndexer(object): + """ + This class contains the commands needed to create, modify and alter the + local-only UID tables for a given mailbox. + + Its purpouse is to keep a local-only index with the messages in each + mailbox, mainly to satisfy the demands of the IMAP specification, but + useful too for any effective listing of the messages in a mailbox. + + Since the incoming mail can be processed at any time in any replica, it's + preferred not to attempt to maintain a global chronological global index. + + These indexes are Message Attributes needed for the IMAP specification (rfc + 3501), although they can be useful for other non-imap store + implementations. + """ + # The uids are expected to be 32-bits values, but the ROWIDs in sqlite + # are 64-bit values. I *don't* think it really matters for any + # practical use, but it's good to remmeber we've got that difference going + # on. + + store = None + table_preffix = "leapmail_uid_" + + def __init__(self, store): + self.store = store + + def _query(self, *args, **kw): + assert self.store is not None + return self.store.raw_sqlcipher_query(*args, **kw) + + def create_table(self, mailbox): + """ + Create the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + assert mailbox + sql = ("CREATE TABLE if not exists {preffix}{name}( " + "uid INTEGER PRIMARY KEY AUTOINCREMENT, " + "hash TEXT UNIQUE NOT NULL)".format( + preffix=self.table_preffix, name=mailbox)) + return self._query(sql) + + def delete_table(self, mailbox): + """ + Delete the UID table for a given mailbox. + :param mailbox: the mailbox name + :type mailbox: str + :rtype: Deferred + """ + assert mailbox + sql = ("DROP TABLE if exists {preffix}{name}".format( + preffix=self.table_preffix, name=mailbox)) + return self._query(sql) + + def rename_table(self, oldmailbox, newmailbox): + """ + Delete the UID table for a given mailbox. + :param oldmailbox: the old mailbox name + :type oldmailbox: str + :param newmailbox: the new mailbox name + :type newmailbox: str + :rtype: Deferred + """ + assert oldmailbox + assert newmailbox + assert oldmailbox != newmailbox + sql = ("ALTER TABLE {preffix}{old} " + "RENAME TO {preffix}{new}".format( + preffix=self.table_preffix, + old=oldmailbox, new=newmailbox)) + return self._query(sql) + + def insert_doc(self, mailbox, doc_id): + """ + Insert the doc_id for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M++ + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + assert mailbox + assert doc_id + + if not re.findall(METAMSGID_RE.format(mbox=mailbox), doc_id): + raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") + + def get_rowid(result): + return result[0][0] + + sql = ("INSERT INTO {preffix}{name} VALUES (" + "NULL, ?)".format( + preffix=self.table_preffix, name=mailbox)) + values = (doc_id,) + + sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=mailbox) + d = self._query(sql, values) + d.addCallback(lambda _: self._query(sql_last)) + d.addCallback(get_rowid) + return d + + def delete_doc_by_uid(self, mailbox, uid): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :param uid: the UID of the message. + :type uid: int + :rtype: Deferred + """ + assert mailbox + assert uid + sql = ("DELETE FROM {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (uid,) + return self._query(sql, values) + + def delete_doc_by_hash(self, mailbox, doc_id): + """ + Delete the entry for a MetaMsg in the UID table for a given mailbox. + + The doc_id must be in the format: + + M++ + + :param mailbox: the mailbox name + :type mailbox: str + :param doc_id: the doc_id for the MetaMsg + :type doc_id: str + :return: a deferred that will fire with the uid of the newly inserted + document. + :rtype: Deferred + """ + assert mailbox + assert doc_id + sql = ("DELETE FROM {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (doc_id,) + return self._query(sql, values) + + def get_doc_id_from_uid(self, mailbox, uid): + """ + Get the doc_id for a MetaMsg in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :param uid: the uid for the MetaMsg for this mailbox + :type uid: int + :rtype: Deferred + """ + def get_hash(result): + return result[0][0] + + sql = ("SELECT hash from {preffix}{name} " + "WHERE uid=?".format( + preffix=self.table_preffix, name=mailbox)) + values = (uid,) + d = self._query(sql, values) + d.addCallback(get_hash) + return d + + def get_doc_ids_from_uids(self, mailbox, uids): + # For IMAP relative numbering /sequences. + # XXX dereference the range (n,*) + raise NotImplementedError() + + def count(self, mailbox): + """ + Get the number of entries in the UID table for a given mailbox. + + :param mailbox: the mailbox name + :type mailbox: str + :return: a deferred that will fire with an integer returning the count. + :rtype: Deferred + """ + def get_count(result): + return result[0][0] + + sql = ("SELECT Count(*) FROM {preffix}{name};".format( + preffix=self.table_preffix, name=mailbox)) + d = self._query(sql) + d.addCallback(get_count) + return d + + def get_next_uid(self, mailbox): + """ + Get the next integer beyond the highest UID count for a given mailbox. + + This is expected by the IMAP implementation. There are no guarantees + that a document to be inserted in the future gets the returned UID: the + only thing that can be assured is that it will be equal or greater than + the value returned. + + :param mailbox: the mailbox name + :type mailbox: str + :return: a deferred that will fire with an integer returning the next + uid. + :rtype: Deferred + """ + assert mailbox + + def increment(result): + return result[0][0] + 1 + + sql = ("SELECT MAX(rowid) FROM {preffix}{name} " + "LIMIT 1;").format( + preffix=self.table_preffix, name=mailbox) + + d = self._query(sql) + d.addCallback(increment) + return d -- cgit v1.2.3 From 60eecafcc8a516985f900ef81bc7941a5234930a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 4 Jan 2015 03:37:18 -0400 Subject: tests for mail.mail module: Message --- src/leap/mail/mailbox_indexer.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'src/leap/mail/mailbox_indexer.py') diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index bc298ea..1ceaec0 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -22,6 +22,17 @@ import re from leap.mail.constants import METAMSGID_RE +def _maybe_first_query_item(thing): + """ + Return the first item the returned query result, or None + if empty. + """ + try: + return thing[0][0] + except IndexError: + return None + + class WrongMetaDocIDError(Exception): pass @@ -124,7 +135,7 @@ class MailboxIndexer(object): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): - return result[0][0] + return _maybe_first_query_item(result) sql = ("INSERT INTO {preffix}{name} VALUES (" "NULL, ?)".format( @@ -192,7 +203,7 @@ class MailboxIndexer(object): :rtype: Deferred """ def get_hash(result): - return result[0][0] + return _maybe_first_query_item(result) sql = ("SELECT hash from {preffix}{name} " "WHERE uid=?".format( @@ -217,7 +228,7 @@ class MailboxIndexer(object): :rtype: Deferred """ def get_count(result): - return result[0][0] + return _maybe_first_query_item(result) sql = ("SELECT Count(*) FROM {preffix}{name};".format( preffix=self.table_preffix, name=mailbox)) @@ -243,7 +254,10 @@ class MailboxIndexer(object): assert mailbox def increment(result): - return result[0][0] + 1 + uid = _maybe_first_query_item(result) + if uid is None: + return None + return uid + 1 sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( -- cgit v1.2.3 From 1b457bbe0eefa12d3e75b58247b53cc62aecc356 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 6 Jan 2015 02:19:02 -0400 Subject: tests for mail.mail module: Account --- src/leap/mail/mailbox_indexer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/leap/mail/mailbox_indexer.py') diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 1ceaec0..e5b813f 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -256,7 +256,7 @@ class MailboxIndexer(object): def increment(result): uid = _maybe_first_query_item(result) if uid is None: - return None + return 1 return uid + 1 sql = ("SELECT MAX(rowid) FROM {preffix}{name} " -- cgit v1.2.3 From c1fc9b52d8b577814e921d128357afdbd9278662 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 12 Jan 2015 20:47:29 -0400 Subject: Use mailbox uuids The previous implementation is naive, since it imposes a burden when renaming mailboxes. We're using uuids in the local uid tables instead, which is more cryptic but way more efficient. * receive mbox uuid instead of name * use mailbox uuid in identifiers --- src/leap/mail/mailbox_indexer.py | 101 +++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 47 deletions(-) (limited to 'src/leap/mail/mailbox_indexer.py') diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index e5b813f..6155a7a 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -18,6 +18,7 @@ Local tables to store the message Unique Identifiers for a given mailbox. """ import re +import uuid from leap.mail.constants import METAMSGID_RE @@ -37,6 +38,25 @@ class WrongMetaDocIDError(Exception): pass +def sanitize(mailbox_id): + return mailbox_id.replace("-", "_") + + +def check_good_uuid(mailbox_id): + """ + Check that the passed mailbox identifier is a valid UUID. + :param mailbox_id: the uuid to check + :type mailbox_id: str + :return: None + :raises: AssertionError if a wrong uuid was passed. + """ + try: + uuid.UUID(str(mailbox_id)) + except (AttributeError, ValueError): + raise AssertionError( + "the mbox_id is not a valid uuid: %s" % mailbox_id) + + class MailboxIndexer(object): """ This class contains the commands needed to create, modify and alter the @@ -68,51 +88,33 @@ class MailboxIndexer(object): assert self.store is not None return self.store.raw_sqlcipher_query(*args, **kw) - def create_table(self, mailbox): + def create_table(self, mailbox_id): """ Create the UID table for a given mailbox. - :param mailbox: the mailbox name + :param mailbox: the mailbox identifier. :type mailbox: str :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) sql = ("CREATE TABLE if not exists {preffix}{name}( " "uid INTEGER PRIMARY KEY AUTOINCREMENT, " "hash TEXT UNIQUE NOT NULL)".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) - def delete_table(self, mailbox): + def delete_table(self, mailbox_id): """ Delete the UID table for a given mailbox. :param mailbox: the mailbox name :type mailbox: str :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) sql = ("DROP TABLE if exists {preffix}{name}".format( - preffix=self.table_preffix, name=mailbox)) - return self._query(sql) - - def rename_table(self, oldmailbox, newmailbox): - """ - Delete the UID table for a given mailbox. - :param oldmailbox: the old mailbox name - :type oldmailbox: str - :param newmailbox: the new mailbox name - :type newmailbox: str - :rtype: Deferred - """ - assert oldmailbox - assert newmailbox - assert oldmailbox != newmailbox - sql = ("ALTER TABLE {preffix}{old} " - "RENAME TO {preffix}{new}".format( - preffix=self.table_preffix, - old=oldmailbox, new=newmailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) - def insert_doc(self, mailbox, doc_id): + def insert_doc(self, mailbox_id, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -128,10 +130,11 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) assert doc_id + mailbox_id = mailbox_id.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox=mailbox), doc_id): + if not re.findall(METAMSGID_RE.format(mbox=mailbox_id), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -139,44 +142,44 @@ class MailboxIndexer(object): sql = ("INSERT INTO {preffix}{name} VALUES (" "NULL, ?)".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (doc_id,) sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=mailbox) + preffix=self.table_preffix, name=sanitize(mailbox_id)) d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) d.addCallback(get_rowid) return d - def delete_doc_by_uid(self, mailbox, uid): + def delete_doc_by_uid(self, mailbox_id, uid): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :param uid: the UID of the message. :type uid: int :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) assert uid sql = ("DELETE FROM {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (uid,) return self._query(sql, values) - def delete_doc_by_hash(self, mailbox, doc_id): + def delete_doc_by_hash(self, mailbox_id, doc_id): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. The doc_id must be in the format: - M++ + M-- - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :param doc_id: the doc_id for the MetaMsg :type doc_id: str @@ -184,30 +187,32 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) assert doc_id sql = ("DELETE FROM {preffix}{name} " "WHERE hash=?".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (doc_id,) return self._query(sql, values) - def get_doc_id_from_uid(self, mailbox, uid): + def get_doc_id_from_uid(self, mailbox_id, uid): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox name + :param mailbox: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ + check_good_uuid(mailbox_id) + def get_hash(result): return _maybe_first_query_item(result) sql = ("SELECT hash from {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) values = (uid,) d = self._query(sql, values) d.addCallback(get_hash) @@ -218,7 +223,7 @@ class MailboxIndexer(object): # XXX dereference the range (n,*) raise NotImplementedError() - def count(self, mailbox): + def count(self, mailbox_id): """ Get the number of entries in the UID table for a given mailbox. @@ -227,16 +232,18 @@ class MailboxIndexer(object): :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ + check_good_uuid(mailbox_id) + def get_count(result): return _maybe_first_query_item(result) sql = ("SELECT Count(*) FROM {preffix}{name};".format( - preffix=self.table_preffix, name=mailbox)) + preffix=self.table_preffix, name=sanitize(mailbox_id))) d = self._query(sql) d.addCallback(get_count) return d - def get_next_uid(self, mailbox): + def get_next_uid(self, mailbox_id): """ Get the next integer beyond the highest UID count for a given mailbox. @@ -251,7 +258,7 @@ class MailboxIndexer(object): uid. :rtype: Deferred """ - assert mailbox + check_good_uuid(mailbox_id) def increment(result): uid = _maybe_first_query_item(result) @@ -261,7 +268,7 @@ class MailboxIndexer(object): sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=mailbox) + preffix=self.table_preffix, name=sanitize(mailbox_id)) d = self._query(sql) d.addCallback(increment) -- cgit v1.2.3 From 68500fb15dbb7531eeb397ccee2c160d71284d97 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 7 Jan 2015 12:12:24 -0400 Subject: Complete IMAP implementation, update tests --- src/leap/mail/mailbox_indexer.py | 70 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 6 deletions(-) (limited to 'src/leap/mail/mailbox_indexer.py') diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 6155a7a..22e57d4 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -114,6 +114,24 @@ class MailboxIndexer(object): preffix=self.table_preffix, name=sanitize(mailbox_id))) return self._query(sql) + def rename_table(self, oldmailbox, newmailbox): + """ + Delete the UID table for a given mailbox. + :param oldmailbox: the old mailbox name + :type oldmailbox: str + :param newmailbox: the new mailbox name + :type newmailbox: str + :rtype: Deferred + """ + assert oldmailbox + assert newmailbox + assert oldmailbox != newmailbox + sql = ("ALTER TABLE {preffix}{old} " + "RENAME TO {preffix}{new}".format( + preffix=self.table_preffix, + old=sanitize(oldmailbox), new=sanitize(newmailbox))) + return self._query(sql) + def insert_doc(self, mailbox_id, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -134,7 +152,7 @@ class MailboxIndexer(object): assert doc_id mailbox_id = mailbox_id.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox=mailbox_id), doc_id): + if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_id), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -148,9 +166,11 @@ class MailboxIndexer(object): sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( preffix=self.table_preffix, name=sanitize(mailbox_id)) + d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) d.addCallback(get_rowid) + d.addErrback(lambda f: f.printTraceback()) return d def delete_doc_by_uid(self, mailbox_id, uid): @@ -199,13 +219,14 @@ class MailboxIndexer(object): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox: the mailbox uuid + :param mailbox_id: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ check_good_uuid(mailbox_id) + mailbox_id = mailbox_id.replace('-', '_') def get_hash(result): return _maybe_first_query_item(result) @@ -218,7 +239,24 @@ class MailboxIndexer(object): d.addCallback(get_hash) return d - def get_doc_ids_from_uids(self, mailbox, uids): + def get_uid_from_doc_id(self, mailbox_id, doc_id): + check_good_uuid(mailbox_id) + mailbox_id = mailbox_id.replace('-', '_') + + def get_uid(result): + return _maybe_first_query_item(result) + + sql = ("SELECT uid from {preffix}{name} " + "WHERE hash=?".format( + preffix=self.table_preffix, name=sanitize(mailbox_id))) + values = (doc_id,) + d = self._query(sql, values) + d.addCallback(get_uid) + return d + + + + def get_doc_ids_from_uids(self, mailbox_id, uids): # For IMAP relative numbering /sequences. # XXX dereference the range (n,*) raise NotImplementedError() @@ -227,8 +265,8 @@ class MailboxIndexer(object): """ Get the number of entries in the UID table for a given mailbox. - :param mailbox: the mailbox name - :type mailbox: str + :param mailbox_id: the mailbox uuid + :type mailbox_id: str :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ @@ -241,6 +279,7 @@ class MailboxIndexer(object): preffix=self.table_preffix, name=sanitize(mailbox_id))) d = self._query(sql) d.addCallback(get_count) + d.addErrback(lambda _: 0) return d def get_next_uid(self, mailbox_id): @@ -252,7 +291,7 @@ class MailboxIndexer(object): only thing that can be assured is that it will be equal or greater than the value returned. - :param mailbox: the mailbox name + :param mailbox_id: the mailbox uuid :type mailbox: str :return: a deferred that will fire with an integer returning the next uid. @@ -273,3 +312,22 @@ class MailboxIndexer(object): d = self._query(sql) d.addCallback(increment) return d + + def all_uid_iter(self, mailbox_id): + """ + Get a sequence of all the uids in this mailbox. + + :param mailbox_id: the mailbox uuid + :type mailbox_id: str + """ + check_good_uuid(mailbox_id) + + sql = ("SELECT uid from {preffix}{name} ").format( + preffix=self.table_preffix, name=sanitize(mailbox_id)) + + def get_results(result): + return [x[0] for x in result] + + d = self._query(sql) + d.addCallback(get_results) + return d -- cgit v1.2.3 From 8c65f09a16e4e00452dffa7d72771d9fac21c9c0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 20 Jan 2015 13:48:21 -0400 Subject: imap: complete FETCH implementation --- src/leap/mail/mailbox_indexer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'src/leap/mail/mailbox_indexer.py') diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 22e57d4..43a1f60 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -305,12 +305,24 @@ class MailboxIndexer(object): return 1 return uid + 1 + d = self.get_last_uid(mailbox_id) + d.addCallback(increment) + return d + + def get_last_uid(self, mailbox_id): + """ + Get the highest UID for a given mailbox. + """ + check_good_uuid(mailbox_id) sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( preffix=self.table_preffix, name=sanitize(mailbox_id)) + def getit(result): + return _maybe_first_query_item(result) + d = self._query(sql) - d.addCallback(increment) + d.addCallback(getit) return d def all_uid_iter(self, mailbox_id): -- cgit v1.2.3 From aab87b503184619b5637d6b326ec7717e34dfb99 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 16 Jan 2015 19:20:37 -0400 Subject: lots of little fixes after meskio's review mostly having to do with poor, missing or outdated documentation, naming of confusing things and reordering of code blocks for improved readability. --- src/leap/mail/mailbox_indexer.py | 128 +++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 74 deletions(-) (limited to 'src/leap/mail/mailbox_indexer.py') diff --git a/src/leap/mail/mailbox_indexer.py b/src/leap/mail/mailbox_indexer.py index 43a1f60..4eb0fa8 100644 --- a/src/leap/mail/mailbox_indexer.py +++ b/src/leap/mail/mailbox_indexer.py @@ -38,23 +38,23 @@ class WrongMetaDocIDError(Exception): pass -def sanitize(mailbox_id): - return mailbox_id.replace("-", "_") +def sanitize(mailbox_uuid): + return mailbox_uuid.replace("-", "_") -def check_good_uuid(mailbox_id): +def check_good_uuid(mailbox_uuid): """ Check that the passed mailbox identifier is a valid UUID. - :param mailbox_id: the uuid to check - :type mailbox_id: str + :param mailbox_uuid: the uuid to check + :type mailbox_uuid: str :return: None :raises: AssertionError if a wrong uuid was passed. """ try: - uuid.UUID(str(mailbox_id)) + uuid.UUID(str(mailbox_uuid)) except (AttributeError, ValueError): raise AssertionError( - "the mbox_id is not a valid uuid: %s" % mailbox_id) + "the mbox_id is not a valid uuid: %s" % mailbox_uuid) class MailboxIndexer(object): @@ -88,51 +88,33 @@ class MailboxIndexer(object): assert self.store is not None return self.store.raw_sqlcipher_query(*args, **kw) - def create_table(self, mailbox_id): + def create_table(self, mailbox_uuid): """ Create the UID table for a given mailbox. :param mailbox: the mailbox identifier. :type mailbox: str :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("CREATE TABLE if not exists {preffix}{name}( " "uid INTEGER PRIMARY KEY AUTOINCREMENT, " "hash TEXT UNIQUE NOT NULL)".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) return self._query(sql) - def delete_table(self, mailbox_id): + def delete_table(self, mailbox_uuid): """ Delete the UID table for a given mailbox. :param mailbox: the mailbox name :type mailbox: str :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("DROP TABLE if exists {preffix}{name}".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) return self._query(sql) - def rename_table(self, oldmailbox, newmailbox): - """ - Delete the UID table for a given mailbox. - :param oldmailbox: the old mailbox name - :type oldmailbox: str - :param newmailbox: the new mailbox name - :type newmailbox: str - :rtype: Deferred - """ - assert oldmailbox - assert newmailbox - assert oldmailbox != newmailbox - sql = ("ALTER TABLE {preffix}{old} " - "RENAME TO {preffix}{new}".format( - preffix=self.table_preffix, - old=sanitize(oldmailbox), new=sanitize(newmailbox))) - return self._query(sql) - - def insert_doc(self, mailbox_id, doc_id): + def insert_doc(self, mailbox_uuid, doc_id): """ Insert the doc_id for a MetaMsg in the UID table for a given mailbox. @@ -148,11 +130,11 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) assert doc_id - mailbox_id = mailbox_id.replace('-', '_') + mailbox_uuid = mailbox_uuid.replace('-', '_') - if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_id), doc_id): + if not re.findall(METAMSGID_RE.format(mbox_uuid=mailbox_uuid), doc_id): raise WrongMetaDocIDError("Wrong format for the MetaMsg doc_id") def get_rowid(result): @@ -160,12 +142,12 @@ class MailboxIndexer(object): sql = ("INSERT INTO {preffix}{name} VALUES (" "NULL, ?)".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (doc_id,) sql_last = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=sanitize(mailbox_id)) + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) d = self._query(sql, values) d.addCallback(lambda _: self._query(sql_last)) @@ -173,25 +155,25 @@ class MailboxIndexer(object): d.addErrback(lambda f: f.printTraceback()) return d - def delete_doc_by_uid(self, mailbox_id, uid): + def delete_doc_by_uid(self, mailbox_uuid, uid): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :param uid: the UID of the message. :type uid: int :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) assert uid sql = ("DELETE FROM {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (uid,) return self._query(sql, values) - def delete_doc_by_hash(self, mailbox_id, doc_id): + def delete_doc_by_hash(self, mailbox_uuid, doc_id): """ Delete the entry for a MetaMsg in the UID table for a given mailbox. @@ -199,7 +181,7 @@ class MailboxIndexer(object): M-- - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :param doc_id: the doc_id for the MetaMsg :type doc_id: str @@ -207,82 +189,80 @@ class MailboxIndexer(object): document. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) assert doc_id sql = ("DELETE FROM {preffix}{name} " "WHERE hash=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (doc_id,) return self._query(sql, values) - def get_doc_id_from_uid(self, mailbox_id, uid): + def get_doc_id_from_uid(self, mailbox_uuid, uid): """ Get the doc_id for a MetaMsg in the UID table for a given mailbox. - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :param uid: the uid for the MetaMsg for this mailbox :type uid: int :rtype: Deferred """ - check_good_uuid(mailbox_id) - mailbox_id = mailbox_id.replace('-', '_') + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') def get_hash(result): return _maybe_first_query_item(result) sql = ("SELECT hash from {preffix}{name} " "WHERE uid=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (uid,) d = self._query(sql, values) d.addCallback(get_hash) return d - def get_uid_from_doc_id(self, mailbox_id, doc_id): - check_good_uuid(mailbox_id) - mailbox_id = mailbox_id.replace('-', '_') + def get_uid_from_doc_id(self, mailbox_uuid, doc_id): + check_good_uuid(mailbox_uuid) + mailbox_uuid = mailbox_uuid.replace('-', '_') def get_uid(result): return _maybe_first_query_item(result) sql = ("SELECT uid from {preffix}{name} " "WHERE hash=?".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) values = (doc_id,) d = self._query(sql, values) d.addCallback(get_uid) return d - - - def get_doc_ids_from_uids(self, mailbox_id, uids): + def get_doc_ids_from_uids(self, mailbox_uuid, uids): # For IMAP relative numbering /sequences. # XXX dereference the range (n,*) raise NotImplementedError() - def count(self, mailbox_id): + def count(self, mailbox_uuid): """ Get the number of entries in the UID table for a given mailbox. - :param mailbox_id: the mailbox uuid - :type mailbox_id: str + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str :return: a deferred that will fire with an integer returning the count. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) def get_count(result): return _maybe_first_query_item(result) sql = ("SELECT Count(*) FROM {preffix}{name};".format( - preffix=self.table_preffix, name=sanitize(mailbox_id))) + preffix=self.table_preffix, name=sanitize(mailbox_uuid))) d = self._query(sql) d.addCallback(get_count) d.addErrback(lambda _: 0) return d - def get_next_uid(self, mailbox_id): + def get_next_uid(self, mailbox_uuid): """ Get the next integer beyond the highest UID count for a given mailbox. @@ -291,13 +271,13 @@ class MailboxIndexer(object): only thing that can be assured is that it will be equal or greater than the value returned. - :param mailbox_id: the mailbox uuid + :param mailbox_uuid: the mailbox uuid :type mailbox: str :return: a deferred that will fire with an integer returning the next uid. :rtype: Deferred """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) def increment(result): uid = _maybe_first_query_item(result) @@ -305,18 +285,18 @@ class MailboxIndexer(object): return 1 return uid + 1 - d = self.get_last_uid(mailbox_id) + d = self.get_last_uid(mailbox_uuid) d.addCallback(increment) return d - def get_last_uid(self, mailbox_id): + def get_last_uid(self, mailbox_uuid): """ Get the highest UID for a given mailbox. """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("SELECT MAX(rowid) FROM {preffix}{name} " "LIMIT 1;").format( - preffix=self.table_preffix, name=sanitize(mailbox_id)) + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) def getit(result): return _maybe_first_query_item(result) @@ -325,17 +305,17 @@ class MailboxIndexer(object): d.addCallback(getit) return d - def all_uid_iter(self, mailbox_id): + def all_uid_iter(self, mailbox_uuid): """ Get a sequence of all the uids in this mailbox. - :param mailbox_id: the mailbox uuid - :type mailbox_id: str + :param mailbox_uuid: the mailbox uuid + :type mailbox_uuid: str """ - check_good_uuid(mailbox_id) + check_good_uuid(mailbox_uuid) sql = ("SELECT uid from {preffix}{name} ").format( - preffix=self.table_preffix, name=sanitize(mailbox_id)) + preffix=self.table_preffix, name=sanitize(mailbox_uuid)) def get_results(result): return [x[0] for x in result] -- cgit v1.2.3