From 8d504fa812da93df3a26c4b4b761a74685d40f25 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 24 Dec 2013 08:17:37 -0200 Subject: Avoid concurrent sync attempts from the same replica in the client (#4451). --- client/changes/feature_4451_avoid_concurrent_syncs | 2 ++ client/src/leap/soledad/client/__init__.py | 23 +++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 client/changes/feature_4451_avoid_concurrent_syncs (limited to 'client') diff --git a/client/changes/feature_4451_avoid_concurrent_syncs b/client/changes/feature_4451_avoid_concurrent_syncs new file mode 100644 index 00000000..04a2c4df --- /dev/null +++ b/client/changes/feature_4451_avoid_concurrent_syncs @@ -0,0 +1,2 @@ + o Avoid concurrent syncs for the same account, but allow for distinct + accounts (4451). diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index a0b3f45a..d35d3a2a 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -34,6 +34,8 @@ import urlparse import hmac from hashlib import sha256 +from threading import Lock +from collections import defaultdict try: import cchardet as chardet @@ -245,6 +247,12 @@ class Soledad(object): Prefix for default values for path. """ + syncing_lock = defaultdict(Lock) + """ + A dictionary that hold locks which avoid multiple sync attempts from the + same database replica. + """ + def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, auth_token=None, secret_id=None): """ @@ -1063,6 +1071,9 @@ class Soledad(object): """ Synchronize the local encrypted replica with a remote replica. + This method blocks until a syncing lock is acquired, so there are no + attempts of concurrent syncs from the same client replica. + :param url: the url of the target replica to sync with :type url: str @@ -1071,11 +1082,13 @@ class Soledad(object): :rtype: str """ if self._db: - local_gen = self._db.sync( - urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=True) - signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) - return local_gen + # acquire lock before attempt to sync + with Soledad.syncing_lock[self._db._get_replica_uid()]: + local_gen = self._db.sync( + urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), + creds=self._creds, autocreate=True) + signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) + return local_gen def need_sync(self, url): """ -- cgit v1.2.3 From 89d3e4a1321ff9701ac67933f8e649cfecd1d95e Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jan 2014 10:29:43 -0200 Subject: Add proper error reporting to shared db lock. --- client/changes/bug_4435_catch-lock-timeout-exception | 1 + client/src/leap/soledad/client/__init__.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 client/changes/bug_4435_catch-lock-timeout-exception (limited to 'client') diff --git a/client/changes/bug_4435_catch-lock-timeout-exception b/client/changes/bug_4435_catch-lock-timeout-exception new file mode 100644 index 00000000..12c05685 --- /dev/null +++ b/client/changes/bug_4435_catch-lock-timeout-exception @@ -0,0 +1 @@ + o Catch lock timeout exception (#4435). diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index d35d3a2a..11e8585b 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -54,6 +54,7 @@ from leap.soledad.common.errors import ( InvalidTokenError, NotLockedError, AlreadyLockedError, + LockTimedOutError, ) from leap.soledad.common.crypto import ( MacMethods, @@ -410,6 +411,8 @@ class Soledad(object): token, timeout = self._shared_db.lock() except AlreadyLockedError: raise BootstrapSequenceError('Database is already locked.') + except LockTimedOutError: + raise BootstrapSequenceError('Lock operation timed out.') try: self._get_or_gen_crypto_secrets() -- cgit v1.2.3 From 360019313b371da17241abbd72038bdebe1b2649 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Jan 2014 13:57:32 -0400 Subject: add rw_lock to client ops --- client/changes/feature_rw_lock_client_ops | 1 + client/src/leap/soledad/client/__init__.py | 80 ++++++++++++++++++------------ 2 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 client/changes/feature_rw_lock_client_ops (limited to 'client') diff --git a/client/changes/feature_rw_lock_client_ops b/client/changes/feature_rw_lock_client_ops new file mode 100644 index 00000000..1c1ab446 --- /dev/null +++ b/client/changes/feature_rw_lock_client_ops @@ -0,0 +1 @@ + o Add a read-write lock for all client operations. Addresses: #4972 diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 11e8585b..48c703ed 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -249,6 +249,7 @@ class Soledad(object): """ syncing_lock = defaultdict(Lock) + rw_lock = Lock() """ A dictionary that hold locks which avoid multiple sync attempts from the same database replica. @@ -790,7 +791,8 @@ class Soledad(object): :rtype: str """ doc.content = self._convert_to_unicode(doc.content) - return self._db.put_doc(doc) + with self.rw_lock: + return self._db.put_doc(doc) def delete_doc(self, doc): """ @@ -802,7 +804,8 @@ class Soledad(object): :return: the new revision identifier for the document :rtype: str """ - return self._db.delete_doc(doc) + with self.rw_lock: + return self._db.delete_doc(doc) def get_doc(self, doc_id, include_deleted=False): """ @@ -818,7 +821,8 @@ class Soledad(object): :return: the document object or None :rtype: SoledadDocument """ - return self._db.get_doc(doc_id, include_deleted=include_deleted) + with self.rw_lock: + return self._db.get_doc(doc_id, include_deleted=include_deleted) def get_docs(self, doc_ids, check_for_conflicts=True, include_deleted=False): @@ -835,9 +839,10 @@ class Soledad(object): in matching doc_ids order. :rtype: generator """ - return self._db.get_docs(doc_ids, - check_for_conflicts=check_for_conflicts, - include_deleted=include_deleted) + with self.rw_lock: + return self._db.get_docs( + doc_ids, check_for_conflicts=check_for_conflicts, + include_deleted=include_deleted) def get_all_docs(self, include_deleted=False): """Get the JSON content for all documents in the database. @@ -849,7 +854,8 @@ class Soledad(object): The current generation of the database, followed by a list of all the documents in the database. """ - return self._db.get_all_docs(include_deleted) + with self.rw_lock: + return self._db.get_all_docs(include_deleted) def _convert_to_unicode(self, content): """ @@ -894,8 +900,9 @@ class Soledad(object): :return: the new document :rtype: SoledadDocument """ - return self._db.create_doc( - self._convert_to_unicode(content), doc_id=doc_id) + with self.rw_lock: + return self._db.create_doc( + self._convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): """ @@ -914,7 +921,8 @@ class Soledad(object): :return: The new cocument :rtype: SoledadDocument """ - return self._db.create_doc_from_json(json, doc_id=doc_id) + with self.rw_lock: + return self._db.create_doc_from_json(json, doc_id=doc_id) def create_index(self, index_name, *index_expressions): """ @@ -938,8 +946,10 @@ class Soledad(object): "number(fieldname, width)", "lower(fieldname)" """ - if self._db: - return self._db.create_index(index_name, *index_expressions) + with self.rw_lock: + if self._db: + return self._db.create_index( + index_name, *index_expressions) def delete_index(self, index_name): """ @@ -948,8 +958,9 @@ class Soledad(object): :param index_name: The name of the index we are removing :type index_name: str """ - if self._db: - return self._db.delete_index(index_name) + with self.rw_lock: + if self._db: + return self._db.delete_index(index_name) def list_indexes(self): """ @@ -958,8 +969,9 @@ class Soledad(object): :return: A list of [('index-name', ['field', 'field2'])] definitions. :rtype: list """ - if self._db: - return self._db.list_indexes() + with self.rw_lock: + if self._db: + return self._db.list_indexes() def get_from_index(self, index_name, *key_values): """ @@ -981,8 +993,9 @@ class Soledad(object): :return: List of [Document] :rtype: list """ - if self._db: - return self._db.get_from_index(index_name, *key_values) + with self.rw_lock: + if self._db: + return self._db.get_from_index(index_name, *key_values) def get_count_from_index(self, index_name, *key_values): """ @@ -998,8 +1011,9 @@ class Soledad(object): :return: count. :rtype: int """ - if self._db: - return self._db.get_count_from_index(index_name, *key_values) + with self.rw_lock: + if self._db: + return self._db.get_count_from_index(index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): """ @@ -1028,9 +1042,10 @@ class Soledad(object): :return: List of [Document] :rtype: list """ - if self._db: - return self._db.get_range_from_index( - index_name, start_value, end_value) + with self.rw_lock: + if self._db: + return self._db.get_range_from_index( + index_name, start_value, end_value) def get_index_keys(self, index_name): """ @@ -1041,8 +1056,9 @@ class Soledad(object): :return: [] A list of tuples of indexed keys. :rtype: list """ - if self._db: - return self._db.get_index_keys(index_name) + with self.rw_lock: + if self._db: + return self._db.get_index_keys(index_name) def get_doc_conflicts(self, doc_id): """ @@ -1054,8 +1070,9 @@ class Soledad(object): :return: a list of the document entries that are conflicted :rtype: list """ - if self._db: - return self._db.get_doc_conflicts(doc_id) + with self.rw_lock: + if self._db: + return self._db.get_doc_conflicts(doc_id) def resolve_doc(self, doc, conflicted_doc_revs): """ @@ -1067,8 +1084,9 @@ class Soledad(object): supersedes. :type conflicted_doc_revs: list """ - if self._db: - return self._db.resolve_doc(doc, conflicted_doc_revs) + with self.rw_lock: + if self._db: + return self._db.resolve_doc(doc, conflicted_doc_revs) def sync(self): """ @@ -1209,7 +1227,7 @@ class Soledad(object): """ soledad_assert(self.STORAGE_SECRETS_KEY in data) # check mac of the recovery document - mac_auth = False + #mac_auth = False # XXX ? mac = None if MAC_KEY in data: soledad_assert(data[MAC_KEY] is not None) @@ -1232,7 +1250,7 @@ class Soledad(object): if mac != data[MAC_KEY]: raise WrongMac('Could not authenticate recovery document\'s ' 'contents.') - mac_auth = True + #mac_auth = True # XXX ? # include secrets in the secret pool. secrets = 0 for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items(): -- cgit v1.2.3 From b72ec5f7229a6371894666bb242799d89a72f36c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 26 Jan 2014 18:54:22 -0400 Subject: add pragmas for temp_store=mem and synchronous=off controlled by environmental variables --- client/changes/feature_sqlite-optimization-pragmas | 1 + client/src/leap/soledad/client/sqlcipher.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 client/changes/feature_sqlite-optimization-pragmas (limited to 'client') diff --git a/client/changes/feature_sqlite-optimization-pragmas b/client/changes/feature_sqlite-optimization-pragmas new file mode 100644 index 00000000..7a35f005 --- /dev/null +++ b/client/changes/feature_sqlite-optimization-pragmas @@ -0,0 +1 @@ + o Add sync=off and tem_store=mem to soledad client, for optimization. diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 43c871c3..ef059e9b 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -190,6 +190,10 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self._set_crypto_pragmas( self._db_handle, password, raw_key, cipher, kdf_iter, cipher_page_size) + if os.environ.get('LEAP_SQLITE_NOSYNC'): + self._pragma_synchronous_off(self._db_handle) + if os.environ.get('LEAP_SQLITE_MEMSTORE'): + self._pragma_mem_temp_store(self._db_handle) self._real_replica_uid = None self._ensure_schema() self._crypto = crypto @@ -734,6 +738,22 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # XXX change passphrase param! db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % passphrase) + @classmethod + def _pragma_synchronous_off(cls, db_handle): + """ + Change the setting of the "synchronous" flag to OFF. + """ + logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF") + db_handle.cursor().execute('PRAGMA synchronous=OFF') + + @classmethod + def _pragma_mem_temp_store(cls, db_handle): + """ + Use a in-memory store for temporary tables. + """ + logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY") + db_handle.cursor().execute('PRAGMA temp_store=MEMORY') + # Extra query methods: extensions to the base sqlite implmentation. def get_count_from_index(self, index_name, *key_values): -- cgit v1.2.3 From 27a70fbbde42166c268c60e624ed11eac7788b55 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Mon, 27 Jan 2014 14:49:59 -0300 Subject: Always return unicode, even on UnicodeError. --- client/changes/bug_return-always-unicode | 1 + client/src/leap/soledad/client/__init__.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 client/changes/bug_return-always-unicode (limited to 'client') diff --git a/client/changes/bug_return-always-unicode b/client/changes/bug_return-always-unicode new file mode 100644 index 00000000..f4ee51ed --- /dev/null +++ b/client/changes/bug_return-always-unicode @@ -0,0 +1 @@ + o Always return unicode in helper method, even on UnicodeError. Related to #4998. diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 48c703ed..3fb037c8 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -859,7 +859,7 @@ class Soledad(object): def _convert_to_unicode(self, content): """ - Converts content to utf8 (or all the strings in content) + Converts content to unicode (or all the strings in content) NOTE: Even though this method supports any type, it will currently ignore contents of lists, tuple or any other @@ -874,13 +874,14 @@ class Soledad(object): if isinstance(content, unicode): return content elif isinstance(content, str): + result = chardet.detect(content) + default = "utf-8" + encoding = result["encoding"] or default try: - result = chardet.detect(content) - default = "utf-8" - encoding = result["encoding"] or default content = content.decode(encoding) - except UnicodeError: - pass + except UnicodeError as e: + logger.error("Unicode error: {0!r}. Using 'replace'".format(e)) + content = content.decode(encoding, 'replace') return content else: if isinstance(content, dict): -- cgit v1.2.3 From dfa01cb0518eade316abb12c10bf2dc808745cea Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 14:52:37 -0400 Subject: Remove global client rw lock leap.mail is handling locks in a finer-grained way now, so we do not need to block everything so much --- client/src/leap/soledad/client/__init__.py | 77 ++++++++++++------------------ 1 file changed, 30 insertions(+), 47 deletions(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 3fb037c8..f0abf130 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -249,7 +249,6 @@ class Soledad(object): """ syncing_lock = defaultdict(Lock) - rw_lock = Lock() """ A dictionary that hold locks which avoid multiple sync attempts from the same database replica. @@ -791,8 +790,7 @@ class Soledad(object): :rtype: str """ doc.content = self._convert_to_unicode(doc.content) - with self.rw_lock: - return self._db.put_doc(doc) + return self._db.put_doc(doc) def delete_doc(self, doc): """ @@ -804,8 +802,7 @@ class Soledad(object): :return: the new revision identifier for the document :rtype: str """ - with self.rw_lock: - return self._db.delete_doc(doc) + return self._db.delete_doc(doc) def get_doc(self, doc_id, include_deleted=False): """ @@ -821,8 +818,7 @@ class Soledad(object): :return: the document object or None :rtype: SoledadDocument """ - with self.rw_lock: - return self._db.get_doc(doc_id, include_deleted=include_deleted) + return self._db.get_doc(doc_id, include_deleted=include_deleted) def get_docs(self, doc_ids, check_for_conflicts=True, include_deleted=False): @@ -839,10 +835,9 @@ class Soledad(object): in matching doc_ids order. :rtype: generator """ - with self.rw_lock: - return self._db.get_docs( - doc_ids, check_for_conflicts=check_for_conflicts, - include_deleted=include_deleted) + return self._db.get_docs( + doc_ids, check_for_conflicts=check_for_conflicts, + include_deleted=include_deleted) def get_all_docs(self, include_deleted=False): """Get the JSON content for all documents in the database. @@ -854,8 +849,7 @@ class Soledad(object): The current generation of the database, followed by a list of all the documents in the database. """ - with self.rw_lock: - return self._db.get_all_docs(include_deleted) + return self._db.get_all_docs(include_deleted) def _convert_to_unicode(self, content): """ @@ -901,9 +895,8 @@ class Soledad(object): :return: the new document :rtype: SoledadDocument """ - with self.rw_lock: - return self._db.create_doc( - self._convert_to_unicode(content), doc_id=doc_id) + return self._db.create_doc( + self._convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): """ @@ -922,8 +915,7 @@ class Soledad(object): :return: The new cocument :rtype: SoledadDocument """ - with self.rw_lock: - return self._db.create_doc_from_json(json, doc_id=doc_id) + return self._db.create_doc_from_json(json, doc_id=doc_id) def create_index(self, index_name, *index_expressions): """ @@ -947,10 +939,9 @@ class Soledad(object): "number(fieldname, width)", "lower(fieldname)" """ - with self.rw_lock: - if self._db: - return self._db.create_index( - index_name, *index_expressions) + if self._db: + return self._db.create_index( + index_name, *index_expressions) def delete_index(self, index_name): """ @@ -959,9 +950,8 @@ class Soledad(object): :param index_name: The name of the index we are removing :type index_name: str """ - with self.rw_lock: - if self._db: - return self._db.delete_index(index_name) + if self._db: + return self._db.delete_index(index_name) def list_indexes(self): """ @@ -970,9 +960,8 @@ class Soledad(object): :return: A list of [('index-name', ['field', 'field2'])] definitions. :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.list_indexes() + if self._db: + return self._db.list_indexes() def get_from_index(self, index_name, *key_values): """ @@ -994,9 +983,8 @@ class Soledad(object): :return: List of [Document] :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.get_from_index(index_name, *key_values) + if self._db: + return self._db.get_from_index(index_name, *key_values) def get_count_from_index(self, index_name, *key_values): """ @@ -1012,9 +1000,8 @@ class Soledad(object): :return: count. :rtype: int """ - with self.rw_lock: - if self._db: - return self._db.get_count_from_index(index_name, *key_values) + if self._db: + return self._db.get_count_from_index(index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): """ @@ -1043,10 +1030,9 @@ class Soledad(object): :return: List of [Document] :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.get_range_from_index( - index_name, start_value, end_value) + if self._db: + return self._db.get_range_from_index( + index_name, start_value, end_value) def get_index_keys(self, index_name): """ @@ -1057,9 +1043,8 @@ class Soledad(object): :return: [] A list of tuples of indexed keys. :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.get_index_keys(index_name) + if self._db: + return self._db.get_index_keys(index_name) def get_doc_conflicts(self, doc_id): """ @@ -1071,9 +1056,8 @@ class Soledad(object): :return: a list of the document entries that are conflicted :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.get_doc_conflicts(doc_id) + if self._db: + return self._db.get_doc_conflicts(doc_id) def resolve_doc(self, doc, conflicted_doc_revs): """ @@ -1085,9 +1069,8 @@ class Soledad(object): supersedes. :type conflicted_doc_revs: list """ - with self.rw_lock: - if self._db: - return self._db.resolve_doc(doc, conflicted_doc_revs) + if self._db: + return self._db.resolve_doc(doc, conflicted_doc_revs) def sync(self): """ -- cgit v1.2.3 From 057e7c28f9fc790fa449cef5361fba9dcd5009d1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 19 Feb 2014 23:34:49 -0400 Subject: add locks for create_doc and update_indexes. Closes: #5139 This solves a InterfaceError (sqlite error code 21) we were having with massive concurrent creation/puts. --- client/src/leap/soledad/client/sqlcipher.py | 31 +++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index ef059e9b..d8ba0b79 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -147,6 +147,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): _index_storage_value = 'expand referenced encrypted' k_lock = threading.Lock() + create_doc_lock = threading.Lock() + update_indexes_lock = threading.Lock() _syncer = None def __init__(self, sqlcipher_file, password, document_factory=None, @@ -400,6 +402,22 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') + def create_doc(self, content, doc_id=None): + """ + Create a new document in the local encrypted database. + + :param content: the contents of the new document + :type content: dict + :param doc_id: an optional identifier specifying the document id + :type doc_id: str + + :return: the new document + :rtype: SoledadDocument + """ + with self.create_doc_lock: + return sqlite_backend.SQLitePartialExpandDatabase.create_doc( + self, content, doc_id=doc_id) + def _put_and_update_indexes(self, old_doc, doc): """ Update a document and all indexes related to it. @@ -409,12 +427,13 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :param doc: The new version of the document. :type doc: u1db.Document """ - sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( - self, old_doc, doc) - c = self._db_handle.cursor() - c.execute('UPDATE document SET syncable=? ' - 'WHERE doc_id=?', - (doc.syncable, doc.doc_id)) + with self.update_indexes_lock: + sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( + self, old_doc, doc) + c = self._db_handle.cursor() + c.execute('UPDATE document SET syncable=? ' + 'WHERE doc_id=?', + (doc.syncable, doc.doc_id)) def _get_doc(self, doc_id, check_for_conflicts=False): """ -- cgit v1.2.3 From 2404b07cc015c4ce76425b7bcf1277a6bbfded64 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 19 Feb 2014 23:50:23 -0400 Subject: Set Write-Ahead Logging with autocommit set to 50 pages, a value that will permit fast reads. also set synchronous mode to normal on regular operation. --- client/src/leap/soledad/client/sqlcipher.py | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) (limited to 'client') diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index d8ba0b79..09efa592 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -194,8 +194,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): cipher_page_size) if os.environ.get('LEAP_SQLITE_NOSYNC'): self._pragma_synchronous_off(self._db_handle) + else: + self._pragma_synchronous_normal(self._db_handle) if os.environ.get('LEAP_SQLITE_MEMSTORE'): self._pragma_mem_temp_store(self._db_handle) + self._pragma_write_ahead_logging(self._db_handle) self._real_replica_uid = None self._ensure_schema() self._crypto = crypto @@ -765,6 +768,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF") db_handle.cursor().execute('PRAGMA synchronous=OFF') + @classmethod + def _pragma_synchronous_normal(cls, db_handle): + """ + Change the setting of the "synchronous" flag to NORMAL. + """ + logger.debug("SQLCIPHER: SETTING SYNCHRONOUS NORMAL") + db_handle.cursor().execute('PRAGMA synchronous=NORMAL') + @classmethod def _pragma_mem_temp_store(cls, db_handle): """ @@ -773,6 +784,40 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY") db_handle.cursor().execute('PRAGMA temp_store=MEMORY') + @classmethod + def _pragma_write_ahead_logging(cls, db_handle): + """ + Enable write-ahead logging, and set the autocheckpoint to 50 pages. + + Setting the autocheckpoint to a small value, we make the reads not + suffer too much performance degradation. + + From the sqlite docs: + + "There is a tradeoff between average read performance and average write + performance. To maximize the read performance, one wants to keep the + WAL as small as possible and hence run checkpoints frequently, perhaps + as often as every COMMIT. To maximize write performance, one wants to + amortize the cost of each checkpoint over as many writes as possible, + meaning that one wants to run checkpoints infrequently and let the WAL + grow as large as possible before each checkpoint. The decision of how + often to run checkpoints may therefore vary from one application to + another depending on the relative read and write performance + requirements of the application. The default strategy is to run a + checkpoint once the WAL reaches 1000 pages" + """ + logger.debug("SQLCIPHER: SETTING WRITE-AHEAD LOGGING") + db_handle.cursor().execute('PRAGMA journal_mode=WAL') + # The optimum value can still use a little bit of tuning, but we favor + # small sizes of the WAL file to get fast reads, since we assume that + # the writes will be quick enough to not block too much. + + # TODO + # As a further improvement, we might want to set autocheckpoint to 0 + # here and do the checkpoints manually in a separate thread, to avoid + # any blocks in the main thread (we should run a loopingcall from here) + db_handle.cursor().execute('PRAGMA wal_autocheckpoint=50') + # Extra query methods: extensions to the base sqlite implmentation. def get_count_from_index(self, index_name, *key_values): -- cgit v1.2.3 From c2e4bad20fa17b92591d861dff2f20ca71610319 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 19 Feb 2014 23:56:27 -0400 Subject: changes file --- client/changes/bug_5139-interface-error | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 client/changes/bug_5139-interface-error (limited to 'client') diff --git a/client/changes/bug_5139-interface-error b/client/changes/bug_5139-interface-error new file mode 100644 index 00000000..9127e70b --- /dev/null +++ b/client/changes/bug_5139-interface-error @@ -0,0 +1,2 @@ +o Add lock for create_doc and update_indexes call, + prevents concurrent access to the db. Closes #5139. -- cgit v1.2.3 From dcff2ecd5ca63b09f5e40cca4e18c4660406d5d6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 28 Feb 2014 05:43:27 +0900 Subject: backward-compatibility for socket.create_connection in 2.6 Closes: #5208 --- client/changes/bug_5208_support_socket_26 | 1 + client/src/leap/soledad/client/__init__.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 client/changes/bug_5208_support_socket_26 (limited to 'client') diff --git a/client/changes/bug_5208_support_socket_26 b/client/changes/bug_5208_support_socket_26 new file mode 100644 index 00000000..b44d1d4e --- /dev/null +++ b/client/changes/bug_5208_support_socket_26 @@ -0,0 +1 @@ + o Back-compatibility for socket.create_connection interface in 2.6. Closes: #5208 diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index f0abf130..b5ce7c32 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -1311,9 +1311,17 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection): # derived from httplib.py def connect(self): - "Connect to a host on a given (SSL) port." - sock = socket.create_connection((self.host, self.port), - SOLEDAD_TIMEOUT, self.source_address) + """ + Connect to a host on a given (SSL) port. + """ + try: + source = self.source_address + sock = socket.create_connection((self.host, self.port), + SOLEDAD_TIMEOUT, source) + except AttributeError: + # source_address was introduced in 2.7 + sock = socket.create_connection((self.host, self.port), + SOLEDAD_TIMEOUT) if self._tunnel_host: self.sock = sock self._tunnel() -- cgit v1.2.3 From b479b64691fef819996df0d62630a0fe0b1bb1a8 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 10 Mar 2014 03:41:17 -0400 Subject: minor docstring corrections --- client/src/leap/soledad/client/sqlcipher.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 09efa592..3aea340d 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -91,10 +91,10 @@ def open(path, password, create=True, document_factory=None, crypto=None, database does not already exist. :param path: The filesystem path for the database to open. - :param type: str + :type path: str :param create: True/False, should the database be created if it doesn't already exist? - :param type: bool + :param create: bool :param document_factory: A function that will be called with the same parameters as Document.__init__. :type document_factory: callable @@ -155,20 +155,22 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024): """ - Create a new sqlcipher file. + Connect to an existing SQLCipher database, creating a new sqlcipher + database file if needed. :param sqlcipher_file: The path for the SQLCipher file. :type sqlcipher_file: str :param password: The password that protects the SQLCipher db. :type password: str :param document_factory: A function that will be called with the same - parameters as Document.__init__. + parameters as Document.__init__. :type document_factory: callable :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. + document contents when syncing. :type crypto: soledad.crypto.SoledadCrypto - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. + :param raw_key: Whether password is a raw 64-char hex string or a + passphrase that should be hashed to obtain the + encyrption key. :type raw_key: bool :param cipher: The cipher and mode to use. :type cipher: str -- cgit v1.2.3 From e5664fb1046e71589d0c41cd605761cc642ff28b Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 10 Mar 2014 15:11:02 -0300 Subject: Fix docstring typo. --- client/src/leap/soledad/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index b5ce7c32..a8d68c88 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -778,7 +778,7 @@ class Soledad(object): ============================== WARNING ============================== This method converts the document's contents to unicode in-place. This - meanse that after calling C{put_doc(doc)}, the contents of the + means that after calling C{put_doc(doc)}, the contents of the document, i.e. C{doc.content}, might be different from before the call. ============================== WARNING ============================== -- cgit v1.2.3 From 7fda4c5448cd3566a802777149c511a1ccc2a143 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 12 Mar 2014 17:27:42 -0300 Subject: Do not autocreate remote db when syncing (#5302). --- client/changes/feature_5302_do-not-create-user-db-when-syncing | 2 ++ client/src/leap/soledad/client/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 client/changes/feature_5302_do-not-create-user-db-when-syncing (limited to 'client') diff --git a/client/changes/feature_5302_do-not-create-user-db-when-syncing b/client/changes/feature_5302_do-not-create-user-db-when-syncing new file mode 100644 index 00000000..c3270ab1 --- /dev/null +++ b/client/changes/feature_5302_do-not-create-user-db-when-syncing @@ -0,0 +1,2 @@ + o Do not autocreate remote user database when syncing. Tapicero should make + sure that that db is created when the user is created. Closes #5302. diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index a8d68c88..46e3cd5f 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -1091,7 +1091,7 @@ class Soledad(object): with Soledad.syncing_lock[self._db._get_replica_uid()]: local_gen = self._db.sync( urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=True) + creds=self._creds, autocreate=False) signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) return local_gen -- cgit v1.2.3 From a3fed4d42ab4a7be7bc7ebe86b35805ac73d62de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Fri, 4 Apr 2014 16:34:33 -0300 Subject: Fold in changes --- client/changes/bug_4435_catch-lock-timeout-exception | 1 - client/changes/bug_5139-interface-error | 2 -- client/changes/bug_5208_support_socket_26 | 1 - client/changes/bug_return-always-unicode | 1 - client/changes/bug_reuse-http-connection | 2 -- client/changes/bug_unlock_shared_if_fails | 2 -- client/changes/feature_4451_avoid_concurrent_syncs | 2 -- client/changes/feature_4616_sqlite_count_by_index | 1 - client/changes/feature_5302_do-not-create-user-db-when-syncing | 2 -- client/changes/feature_rw_lock_client_ops | 1 - client/changes/feature_sqlite-optimization-pragmas | 1 - 11 files changed, 16 deletions(-) delete mode 100644 client/changes/bug_4435_catch-lock-timeout-exception delete mode 100644 client/changes/bug_5139-interface-error delete mode 100644 client/changes/bug_5208_support_socket_26 delete mode 100644 client/changes/bug_return-always-unicode delete mode 100644 client/changes/bug_reuse-http-connection delete mode 100644 client/changes/bug_unlock_shared_if_fails delete mode 100644 client/changes/feature_4451_avoid_concurrent_syncs delete mode 100644 client/changes/feature_4616_sqlite_count_by_index delete mode 100644 client/changes/feature_5302_do-not-create-user-db-when-syncing delete mode 100644 client/changes/feature_rw_lock_client_ops delete mode 100644 client/changes/feature_sqlite-optimization-pragmas (limited to 'client') diff --git a/client/changes/bug_4435_catch-lock-timeout-exception b/client/changes/bug_4435_catch-lock-timeout-exception deleted file mode 100644 index 12c05685..00000000 --- a/client/changes/bug_4435_catch-lock-timeout-exception +++ /dev/null @@ -1 +0,0 @@ - o Catch lock timeout exception (#4435). diff --git a/client/changes/bug_5139-interface-error b/client/changes/bug_5139-interface-error deleted file mode 100644 index 9127e70b..00000000 --- a/client/changes/bug_5139-interface-error +++ /dev/null @@ -1,2 +0,0 @@ -o Add lock for create_doc and update_indexes call, - prevents concurrent access to the db. Closes #5139. diff --git a/client/changes/bug_5208_support_socket_26 b/client/changes/bug_5208_support_socket_26 deleted file mode 100644 index b44d1d4e..00000000 --- a/client/changes/bug_5208_support_socket_26 +++ /dev/null @@ -1 +0,0 @@ - o Back-compatibility for socket.create_connection interface in 2.6. Closes: #5208 diff --git a/client/changes/bug_return-always-unicode b/client/changes/bug_return-always-unicode deleted file mode 100644 index f4ee51ed..00000000 --- a/client/changes/bug_return-always-unicode +++ /dev/null @@ -1 +0,0 @@ - o Always return unicode in helper method, even on UnicodeError. Related to #4998. diff --git a/client/changes/bug_reuse-http-connection b/client/changes/bug_reuse-http-connection deleted file mode 100644 index c6cdd9b4..00000000 --- a/client/changes/bug_reuse-http-connection +++ /dev/null @@ -1,2 +0,0 @@ - o Fix a bug in soledad.client.sqlcipher by which we were creating - a new connection for each sync. diff --git a/client/changes/bug_unlock_shared_if_fails b/client/changes/bug_unlock_shared_if_fails deleted file mode 100644 index fc5716e4..00000000 --- a/client/changes/bug_unlock_shared_if_fails +++ /dev/null @@ -1,2 +0,0 @@ - o Unlock shared_db if anything fails in the bootstrap - sequence. Fixes #4702. \ No newline at end of file diff --git a/client/changes/feature_4451_avoid_concurrent_syncs b/client/changes/feature_4451_avoid_concurrent_syncs deleted file mode 100644 index 04a2c4df..00000000 --- a/client/changes/feature_4451_avoid_concurrent_syncs +++ /dev/null @@ -1,2 +0,0 @@ - o Avoid concurrent syncs for the same account, but allow for distinct - accounts (4451). diff --git a/client/changes/feature_4616_sqlite_count_by_index b/client/changes/feature_4616_sqlite_count_by_index deleted file mode 100644 index c7819d38..00000000 --- a/client/changes/feature_4616_sqlite_count_by_index +++ /dev/null @@ -1 +0,0 @@ - o Adds a get_count_by_index to sqlcipher u1db backend. Related to: #4616 diff --git a/client/changes/feature_5302_do-not-create-user-db-when-syncing b/client/changes/feature_5302_do-not-create-user-db-when-syncing deleted file mode 100644 index c3270ab1..00000000 --- a/client/changes/feature_5302_do-not-create-user-db-when-syncing +++ /dev/null @@ -1,2 +0,0 @@ - o Do not autocreate remote user database when syncing. Tapicero should make - sure that that db is created when the user is created. Closes #5302. diff --git a/client/changes/feature_rw_lock_client_ops b/client/changes/feature_rw_lock_client_ops deleted file mode 100644 index 1c1ab446..00000000 --- a/client/changes/feature_rw_lock_client_ops +++ /dev/null @@ -1 +0,0 @@ - o Add a read-write lock for all client operations. Addresses: #4972 diff --git a/client/changes/feature_sqlite-optimization-pragmas b/client/changes/feature_sqlite-optimization-pragmas deleted file mode 100644 index 7a35f005..00000000 --- a/client/changes/feature_sqlite-optimization-pragmas +++ /dev/null @@ -1 +0,0 @@ - o Add sync=off and tem_store=mem to soledad client, for optimization. -- cgit v1.2.3