From e91ad8c99c7abc99a97b8afb6ac7e3b7e729b219 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 9 Dec 2013 16:57:55 -0400 Subject: pep8 happy --- client/src/leap/soledad/client/_version.py | 35 +++++++++++++++++------------- client/src/leap/soledad/client/auth.py | 1 - 2 files changed, 20 insertions(+), 16 deletions(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/_version.py b/client/src/leap/soledad/client/_version.py index 8db26fe5..963de9bc 100644 --- a/client/src/leap/soledad/client/_version.py +++ b/client/src/leap/soledad/client/_version.py @@ -17,6 +17,7 @@ git_full = "$Format:%H$" import subprocess import sys + def run_command(args, cwd=None, verbose=False): try: # remember shell=False, so use git.cmd on windows, not just git @@ -36,11 +37,10 @@ def run_command(args, cwd=None, verbose=False): return None return stdout - -import sys import re import os.path + def get_expanded_variables(versionfile_source): # the code embedded in _version.py can just fetch the value of these # variables. When used from setup.py, we don't want to import @@ -48,7 +48,7 @@ def get_expanded_variables(versionfile_source): # used from _version.py. variables = {} try: - f = open(versionfile_source,"r") + f = open(versionfile_source, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) @@ -63,12 +63,13 @@ def get_expanded_variables(versionfile_source): pass return variables + def versions_from_expanded_variables(variables, tag_prefix, verbose=False): refnames = variables["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + return {} # unexpanded, so not in an unpacked git-archive tarball refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -93,13 +94,14 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } + return {"version": r, + "full": variables["full"].strip()} # no suitable tags, so we use the full revision id if verbose: print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } + return {"version": variables["full"].strip(), + "full": variables["full"].strip()} + def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): # this runs 'git' from the root of the source tree. That either means @@ -116,7 +118,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): here = os.path.abspath(__file__) except NameError: # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct + return {} # not always correct # versionfile_source is the relative path from the top of the source tree # (where the .git directory might live) to this file. Invert this to find @@ -141,7 +143,8 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): return {} if not stdout.startswith(tag_prefix): if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) + print("tag '%s' doesn't start with prefix '%s'" % + (stdout, tag_prefix)) return {} tag = stdout[len(tag_prefix):] stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) @@ -153,7 +156,8 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): return {"version": tag, "full": full} -def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False): +def versions_from_parentdir(parentdir_prefix, versionfile_source, + verbose=False): if IN_LONG_VERSION_PY: # We're running from _version.py. If it's from a source tree # (execute-in-place), we can work upwards to find the root of the @@ -163,7 +167,7 @@ def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False) here = os.path.abspath(__file__) except NameError: # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + return {} # without __file__, we have no hope # versionfile_source is the relative path from the top of the source # tree to _version.py. Invert this to find the root from __file__. root = here @@ -180,7 +184,8 @@ def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False) dirname = os.path.basename(root) if not dirname.startswith(parentdir_prefix): if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % + print("guessing rootdir is '%s', but '%s' doesn't start " + "with prefix '%s'" % (root, dirname, parentdir_prefix)) return None return {"version": dirname[len(parentdir_prefix):], "full": ""} @@ -189,8 +194,9 @@ tag_prefix = "" parentdir_prefix = "leap.soledad.client-" versionfile_source = "src/leap/soledad/client/_version.py" + def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = { "refnames": git_refnames, "full": git_full } + variables = {"refnames": git_refnames, "full": git_full} ver = versions_from_expanded_variables(variables, tag_prefix, verbose) if not ver: ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) @@ -200,4 +206,3 @@ def get_versions(default={"version": "unknown", "full": ""}, verbose=False): if not ver: ver = default return ver - diff --git a/client/src/leap/soledad/client/auth.py b/client/src/leap/soledad/client/auth.py index d85e3ba6..72ab0008 100644 --- a/client/src/leap/soledad/client/auth.py +++ b/client/src/leap/soledad/client/auth.py @@ -68,4 +68,3 @@ class TokenBasedAuth(object): else: raise errors.UnknownAuthMethod( 'Wrong credentials: %s' % self._creds) - -- cgit v1.2.3 From 221beb86213e55e33953fdd72ff515bdf158840e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Tue, 10 Dec 2013 14:40:08 -0300 Subject: Unlock shared db if bootstrapping fails in any way --- client/changes/bug_unlock_shared_if_fails | 2 + client/src/leap/soledad/client/__init__.py | 84 ++++++++++++++++++------------ 2 files changed, 53 insertions(+), 33 deletions(-) create mode 100644 client/changes/bug_unlock_shared_if_fails (limited to 'client') diff --git a/client/changes/bug_unlock_shared_if_fails b/client/changes/bug_unlock_shared_if_fails new file mode 100644 index 00000000..fc5716e4 --- /dev/null +++ b/client/changes/bug_unlock_shared_if_fails @@ -0,0 +1,2 @@ + o Unlock shared_db if anything fails in the bootstrap + sequence. Fixes #4702. \ No newline at end of file diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 62f93b3d..4c1a4195 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -315,6 +315,47 @@ class Soledad(object): # initialization/destruction methods # + def _get_or_gen_crypto_secrets(self): + """ + Retrieves or generates the crypto secrets. + + Might raise BootstrapSequenceError + """ + doc = self._get_secrets_from_shared_db() + + if doc: + logger.info( + 'Found cryptographic secrets in shared recovery ' + 'database.') + _, mac = self.import_recovery_document(doc.content) + if mac is False: + self.put_secrets_in_shared_db() + self._store_secrets() # save new secrets in local file + if self._secret_id is None: + self._set_secret_id(self._secrets.items()[0][0]) + else: + # STAGE 3 - there are no secrets in server also, so + # generate a secret and store it in remote db. + logger.info( + 'No cryptographic secrets found, creating new ' + ' secrets...') + self._set_secret_id(self._gen_secret()) + try: + self._put_secrets_in_shared_db() + except Exception as ex: + # storing generated secret in shared db failed for + # some reason, so we erase the generated secret and + # raise. + try: + os.unlink(self._secrets_path) + except OSError as e: + if e.errno != errno.ENOENT: # no such file or directory + logger.exception(e) + logger.exception(ex) + raise BootstrapSequenceError( + 'Could not store generated secret in the shared ' + 'database, bailing out...') + def _bootstrap(self): """ Bootstrap local Soledad instance. @@ -342,6 +383,8 @@ class Soledad(object): self._init_dirs() self._crypto = SoledadCrypto(self) + secrets_problem = None + # STAGE 1 - verify if secrets exist locally if not self._has_secret(): # try to load from local storage. @@ -360,38 +403,10 @@ class Soledad(object): except AlreadyLockedError: raise BootstrapSequenceError('Database is already locked.') - doc = self._get_secrets_from_shared_db() - if doc: - logger.info( - 'Found cryptographic secrets in shared recovery ' - 'database.') - _, mac = self.import_recovery_document(doc.content) - if mac is False: - self.put_secrets_in_shared_db() - self._store_secrets() # save new secrets in local file - if self._secret_id is None: - self._set_secret_id(self._secrets.items()[0][0]) - else: - # STAGE 3 - there are no secrets in server also, so - # generate a secret and store it in remote db. - logger.info( - 'No cryptographic secrets found, creating new ' - ' secrets...') - self._set_secret_id(self._gen_secret()) - try: - self._put_secrets_in_shared_db() - except Exception: - # storing generated secret in shared db failed for - # some reason, so we erase the generated secret and - # raise. - try: - os.unlink(self._secrets_path) - except OSError as e: - if errno == 2: # no such file or directory - pass - raise BootstrapSequenceError( - 'Could not store generated secret in the shared ' - 'database, bailing out...') + try: + self._get_or_gen_crypto_secrets() + except Exception as e: + secrets_problem = e # release the lock on shared db try: @@ -416,7 +431,10 @@ class Soledad(object): # --- end of atomic operation in shared db --- # STAGE 4 - local database initialization - self._init_db() + if secrets_problem is None: + self._init_db() + else: + raise secrets_problem def _init_dirs(self): """ -- cgit v1.2.3 From b1a5a88a5f53ac9ff5a56625620b45e949404a99 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 13 Dec 2013 05:16:20 -0400 Subject: get_count_from_index --- client/changes/feature_4616_sqlite_count_by_index | 1 + client/src/leap/soledad/client/__init__.py | 17 ++++++++ client/src/leap/soledad/client/sqlcipher.py | 53 ++++++++++++++++++++++- 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 client/changes/feature_4616_sqlite_count_by_index (limited to 'client') diff --git a/client/changes/feature_4616_sqlite_count_by_index b/client/changes/feature_4616_sqlite_count_by_index new file mode 100644 index 00000000..c7819d38 --- /dev/null +++ b/client/changes/feature_4616_sqlite_count_by_index @@ -0,0 +1 @@ + o Adds a get_count_by_index to sqlcipher u1db backend. Related to: #4616 diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 62f93b3d..0d36c247 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -955,6 +955,23 @@ class Soledad(object): if self._db: return self._db.get_from_index(index_name, *key_values) + def get_count_from_index(self, index_name, *key_values): + """ + Return the count of the documents that match the keys and + values supplied. + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: count. + :rtype: int + """ + if self._db: + return self._db.get_count_from_index(index_name, *key_values) + def get_range_from_index(self, index_name, start_value, end_value): """ Return documents that fall within the specified range. diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 894c6f97..c7aebbe6 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -49,8 +49,8 @@ import time import string import threading - from u1db.backends import sqlite_backend +from u1db import errors from pysqlcipher import dbapi2 from u1db import errors as u1db_errors from leap.soledad.common.document import SoledadDocument @@ -697,6 +697,57 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # XXX change passphrase param! db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % passphrase) + # Extra query methods: extensions to the base sqlite implmentation. + + def get_count_from_index(self, index_name, *key_values): + """ + Returns the count for a given combination of index_name + and key values. + + Extension method made from similar methods in u1db version 13.09 + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: count. + :rtype: int + """ + c = self._db_handle.cursor() + definition = self._get_index_definition(index_name) + + if len(key_values) != len(definition): + raise errors.InvalidValueForIndex() + tables = ["document_fields d%d" % i for i in range(len(definition))] + novalue_where = ["d.doc_id = d%d.doc_id" + " AND d%d.field_name = ?" + % (i, i) for i in range(len(definition))] + exact_where = [novalue_where[i] + + (" AND d%d.value = ?" % (i,)) + for i in range(len(definition))] + args = [] + where = [] + for idx, (field, value) in enumerate(zip(definition, key_values)): + args.append(field) + where.append(exact_where[idx]) + args.append(value) + + tables = ["document_fields d%d" % i for i in range(len(definition))] + statement = ( + "SELECT COUNT(*) FROM document d, %s WHERE %s " % ( + ', '.join(tables), + ' AND '.join(where), + )) + try: + c.execute(statement, tuple(args)) + except dbapi2.OperationalError, e: + raise dbapi2.OperationalError( + str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) + res = c.fetchall() + return res[0][0] + def __del__(self): """ Closes db_handle upon object destruction. -- cgit v1.2.3 From 45241ec8618d24b61b768bc7027473b52945609f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 16 Dec 2013 13:21:19 -0400 Subject: patch the _version file so it reports the running version until now this was only possible when running python setup.py version from the source tree. now the .__version__ also reports correctly the runnng version --- client/src/leap/soledad/client/_version.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/_version.py b/client/src/leap/soledad/client/_version.py index 963de9bc..65efb714 100644 --- a/client/src/leap/soledad/client/_version.py +++ b/client/src/leap/soledad/client/_version.py @@ -128,7 +128,16 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): for i in range(len(versionfile_source.split("/"))): root = os.path.dirname(root) else: - root = os.path.dirname(here) + root = os.path.dirname( + os.path.join('..', here)) + + ###################################################### + # XXX patch for our specific configuration with + # the three projects leap.soledad.{common, client, server} + # inside the same repo. + ###################################################### + root = os.path.dirname(os.path.join('..', root)) + if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) -- cgit v1.2.3 From bbf09fcabd311aa91085cedcbd2e9fbbeebc2526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Tue, 17 Dec 2013 15:53:05 -0300 Subject: Use git.exe instead of git.cmd in windows since we use GitBash --- client/versioneer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'client') diff --git a/client/versioneer.py b/client/versioneer.py index b43ab062..18dfd923 100644 --- a/client/versioneer.py +++ b/client/versioneer.py @@ -115,7 +115,7 @@ import sys def run_command(args, cwd=None, verbose=False): try: - # remember shell=False, so use git.cmd on windows, not just git + # remember shell=False, so use git.exe on windows, not just git p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) except EnvironmentError: e = sys.exc_info()[1] @@ -230,7 +230,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): GIT = "git" if sys.platform == "win32": - GIT = "git.cmd" + GIT = "git.exe" stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], cwd=root) if stdout is None: @@ -305,7 +305,7 @@ import sys def run_command(args, cwd=None, verbose=False): try: - # remember shell=False, so use git.cmd on windows, not just git + # remember shell=False, so use git.exe on windows, not just git p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) except EnvironmentError: e = sys.exc_info()[1] @@ -430,7 +430,7 @@ def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): GIT = "git" if sys.platform == "win32": - GIT = "git.cmd" + GIT = "git.exe" stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], cwd=root) if stdout is None: @@ -486,7 +486,7 @@ import sys def do_vcs_install(versionfile_source, ipy): GIT = "git" if sys.platform == "win32": - GIT = "git.cmd" + GIT = "git.exe" run_command([GIT, "add", "versioneer.py"]) run_command([GIT, "add", versionfile_source]) run_command([GIT, "add", ipy]) -- cgit v1.2.3 From ecc05503b393844ee8b864db9d32db288b875d11 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 18 Dec 2013 12:48:00 -0400 Subject: add http-encoding header, decode it too --- client/src/leap/soledad/client/target.py | 120 ++++++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 9 deletions(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index d8899a97..73f719fb 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -14,22 +14,26 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ A U1DB backend for encrypting data before sending to server and decrypting after receiving. """ - -import simplejson as json +import binascii +import cStringIO +import gzip import hashlib import hmac -import binascii +import logging +import urllib +import simplejson as json +from time import sleep -from u1db.remote import utils +from u1db.remote import utils, http_errors from u1db.errors import BrokenSyncStream +from u1db import errors from u1db.remote.http_target import HTTPSyncTarget +from u1db.remote.http_client import _encode_query_parameter from leap.soledad.common import soledad_assert @@ -53,6 +57,7 @@ from leap.soledad.client.crypto import ( UnknownEncryptionMethod, ) +logger = logging.getLogger(__name__) # # Exceptions @@ -222,6 +227,24 @@ def decrypt_doc(crypto, doc): return plainjson +def _gunzip(data): + """ + Uncompress data that is gzipped. + + :param data: gzipped data + :type data: basestring + """ + buffer = cStringIO.StringIO() + buffer.write(data) + buffer.seek(0) + try: + data = gzip.GzipFile(mode='r', fileobj=buffer).read() + except Exception: + logger.warning("Error while decrypting gzipped data") + buffer.close() + return data + + # # SoledadSyncTarget # @@ -353,6 +376,82 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): raise BrokenSyncStream return res + def _request(self, method, url_parts, params=None, body=None, + content_type=None): + """ + Overloaded method. See u1db docs. + Patched for adding gzip encoding. + """ + + self._ensure_connection() + unquoted_url = url_query = self._url.path + if url_parts: + if not url_query.endswith('/'): + url_query += '/' + unquoted_url = url_query + url_query += '/'.join(urllib.quote(part, safe='') + for part in url_parts) + # oauth performs its own quoting + unquoted_url += '/'.join(url_parts) + encoded_params = {} + if params: + for key, value in params.items(): + key = unicode(key).encode('utf-8') + encoded_params[key] = _encode_query_parameter(value) + url_query += ('?' + urllib.urlencode(encoded_params)) + if body is not None and not isinstance(body, basestring): + body = json.dumps(body) + content_type = 'application/json' + headers = {} + if content_type: + headers['content-type'] = content_type + + # Patched: We would like to receive gzip pretty please + # ---------------------------------------------------- + headers['accept-encoding'] = "gzip" + # ---------------------------------------------------- + + headers.update( + self._sign_request(method, unquoted_url, encoded_params)) + + for delay in self._delays: + try: + self._conn.request(method, url_query, body, headers) + return self._response() + except errors.Unavailable, e: + sleep(delay) + raise e + + def _response(self): + """ + Overloaded method, see u1db docs. + We patched it for decrypting gzip content. + """ + resp = self._conn.getresponse() + body = resp.read() + headers = dict(resp.getheaders()) + + # Patched: We would like to decode gzip + # ---------------------------------------------------- + encoding = headers.get('content-encoding', '') + if "gzip" in encoding: + body = _gunzip(body) + # ---------------------------------------------------- + + if resp.status in (200, 201): + return body, headers + elif resp.status in http_errors.ERROR_STATUSES: + try: + respdic = json.loads(body) + except ValueError: + pass + else: + self._error(respdic) + # special case + if resp.status == 503: + raise errors.Unavailable(body, headers) + raise errors.HTTPError(resp.status, body, headers) + def sync_exchange(self, docs_by_generations, source_replica_uid, last_known_generation, last_known_trans_id, return_doc_cb, ensure_callback=None): @@ -364,8 +463,9 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): syncing. :param docs_by_generations: A list of (doc_id, generation, trans_id) - of local documents that were changed since the last local - generation the remote replica knows about. + of local documents that were changed since + the last local generation the remote + replica knows about. :type docs_by_generations: list of tuples :param source_replica_uid: The uid of the source replica. :type source_replica_uid: str @@ -391,6 +491,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): self._conn.putheader('content-type', 'application/x-u1db-sync-stream') for header_name, header_value in self._sign_request('POST', url, {}): self._conn.putheader(header_name, header_value) + self._conn.putheader('accept-encoding', 'gzip') entries = ['['] size = 1 @@ -428,7 +529,8 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): for entry in entries: self._conn.send(entry) entries = None - data, _ = self._response() + data, headers = self._response() + res = self._parse_sync_stream(data, return_doc_cb, ensure_callback) data = None return res['new_generation'], res['new_transaction_id'] -- cgit v1.2.3 From 7a6d913fed8fc8332174c7ac5c8a4a5472a478c3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 19 Dec 2013 21:52:12 -0400 Subject: Do not instantiate the synchronizer each time. This has the nice effect of letting the persistent-connection reuse the existing connection, avoiding the ssl handshake overhead each time we try to synchronize. This can be traced by logging the instantiation of HttpClientBase in u1db.remote I *think* we should be fine with the timeouts as long as we keep the sync period along the 1 min we are doing now. For other cases, we should look into how to override the default timeout in httplib (used by u1db http_client). --- client/changes/bug_reuse-http-connection | 2 ++ client/src/leap/soledad/client/sqlcipher.py | 43 ++++++++++++++++++++++------- client/src/leap/soledad/client/target.py | 1 + 3 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 client/changes/bug_reuse-http-connection (limited to 'client') diff --git a/client/changes/bug_reuse-http-connection b/client/changes/bug_reuse-http-connection new file mode 100644 index 00000000..c6cdd9b4 --- /dev/null +++ b/client/changes/bug_reuse-http-connection @@ -0,0 +1,2 @@ + o Fix a bug in soledad.client.sqlcipher by which we were creating + a new connection for each sync. diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index c7aebbe6..5695bf82 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -49,10 +49,12 @@ import time import string import threading -from u1db.backends import sqlite_backend -from u1db import errors from pysqlcipher import dbapi2 +from u1db.backends import sqlite_backend +from u1db.sync import Synchronizer from u1db import errors as u1db_errors + +from leap.soledad.client.target import SoledadSyncTarget from leap.soledad.common.document import SoledadDocument logger = logging.getLogger(__name__) @@ -144,6 +146,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): _index_storage_value = 'expand referenced encrypted' k_lock = threading.Lock() + _syncer = None def __init__(self, sqlcipher_file, password, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', @@ -336,13 +339,33 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :return: The local generation before the synchronisation was performed. :rtype: int """ - from u1db.sync import Synchronizer - from leap.soledad.client.target import SoledadSyncTarget - return Synchronizer( - self, - SoledadSyncTarget(url, - creds=creds, - crypto=self._crypto)).sync(autocreate=autocreate) + if not self.syncer: + self._create_syncer(url, creds=creds) + return self.syncer.sync(autocreate=autocreate) + + @property + def syncer(self): + """ + Accesor for synchronizer. + """ + return self._syncer + + def _create_syncer(self, url, creds=None): + """ + Creates a synchronizer + + :param url: The url of the target replica to sync with. + :type url: str + :param creds: optional dictionary giving credentials. + to authorize the operation with the server. + :type creds: dict + """ + if self._syncer is None: + self._syncer = Synchronizer( + self, + SoledadSyncTarget(url, + creds=creds, + crypto=self._crypto)) def _extra_schema_init(self, c): """ @@ -719,7 +742,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): definition = self._get_index_definition(index_name) if len(key_values) != len(definition): - raise errors.InvalidValueForIndex() + raise u1db_errors.InvalidValueForIndex() tables = ["document_fields d%d" % i for i in range(len(definition))] novalue_where = ["d.doc_id = d%d.doc_id" " AND d%d.field_name = ?" diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 73f719fb..3b3d6870 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -63,6 +63,7 @@ logger = logging.getLogger(__name__) # Exceptions # + class DocumentNotEncrypted(Exception): """ Raised for failures in document encryption. -- cgit v1.2.3 From 3a38c1b92e6f1e538f682e4e0f7e99a18f41d7d0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 19 Dec 2013 21:54:25 -0400 Subject: Set soledad timeout default to 2 min --- client/src/leap/soledad/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 61337680..a0b3f45a 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -1283,7 +1283,7 @@ class Soledad(object): #----------------------------------------------------------------------------- # We need a more reasonable timeout (in seconds) -SOLEDAD_TIMEOUT = 10 +SOLEDAD_TIMEOUT = 120 class VerifiedHTTPSConnection(httplib.HTTPSConnection): -- cgit v1.2.3 From 01557be5c80833b27df46f4eab17b29c53a7245b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 20 Dec 2013 11:33:29 -0400 Subject: catch cannotsendrequest exception --- client/src/leap/soledad/client/sqlcipher.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 5695bf82..43c871c3 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -43,11 +43,12 @@ So, as the statements above were introduced for backwards compatibility with SLCipher 1.1 databases, we do not implement them as all SQLCipher databases handled by Soledad should be created by SQLCipher >= 2.0. """ +import httplib import logging import os -import time import string import threading +import time from pysqlcipher import dbapi2 from u1db.backends import sqlite_backend @@ -341,7 +342,20 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ if not self.syncer: self._create_syncer(url, creds=creds) - return self.syncer.sync(autocreate=autocreate) + + try: + res = self.syncer.sync(autocreate=autocreate) + except httplib.CannotSendRequest: + # raised when you reuse httplib.HTTP object for new request + # while you havn't called its getresponse() + # this catch works for the current connclass used + # by our HTTPClientBase, since it uses httplib. + # we will have to replace it if it changes. + logger.info("Replacing connection and trying again...") + self._syncer = None + self._create_syncer(url, creds=creds) + res = self.syncer.sync(autocreate=autocreate) + return res @property def syncer(self): -- cgit v1.2.3 From 8d504fa812da93df3a26c4b4b761a74685d40f25 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 24 Dec 2013 08:17:37 -0200 Subject: Avoid concurrent sync attempts from the same replica in the client (#4451). --- client/changes/feature_4451_avoid_concurrent_syncs | 2 ++ client/src/leap/soledad/client/__init__.py | 23 +++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 client/changes/feature_4451_avoid_concurrent_syncs (limited to 'client') diff --git a/client/changes/feature_4451_avoid_concurrent_syncs b/client/changes/feature_4451_avoid_concurrent_syncs new file mode 100644 index 00000000..04a2c4df --- /dev/null +++ b/client/changes/feature_4451_avoid_concurrent_syncs @@ -0,0 +1,2 @@ + o Avoid concurrent syncs for the same account, but allow for distinct + accounts (4451). diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index a0b3f45a..d35d3a2a 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -34,6 +34,8 @@ import urlparse import hmac from hashlib import sha256 +from threading import Lock +from collections import defaultdict try: import cchardet as chardet @@ -245,6 +247,12 @@ class Soledad(object): Prefix for default values for path. """ + syncing_lock = defaultdict(Lock) + """ + A dictionary that hold locks which avoid multiple sync attempts from the + same database replica. + """ + def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, auth_token=None, secret_id=None): """ @@ -1063,6 +1071,9 @@ class Soledad(object): """ Synchronize the local encrypted replica with a remote replica. + This method blocks until a syncing lock is acquired, so there are no + attempts of concurrent syncs from the same client replica. + :param url: the url of the target replica to sync with :type url: str @@ -1071,11 +1082,13 @@ class Soledad(object): :rtype: str """ if self._db: - local_gen = self._db.sync( - urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=True) - signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) - return local_gen + # acquire lock before attempt to sync + with Soledad.syncing_lock[self._db._get_replica_uid()]: + local_gen = self._db.sync( + urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), + creds=self._creds, autocreate=True) + signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) + return local_gen def need_sync(self, url): """ -- cgit v1.2.3 From 89d3e4a1321ff9701ac67933f8e649cfecd1d95e Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jan 2014 10:29:43 -0200 Subject: Add proper error reporting to shared db lock. --- client/changes/bug_4435_catch-lock-timeout-exception | 1 + client/src/leap/soledad/client/__init__.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 client/changes/bug_4435_catch-lock-timeout-exception (limited to 'client') diff --git a/client/changes/bug_4435_catch-lock-timeout-exception b/client/changes/bug_4435_catch-lock-timeout-exception new file mode 100644 index 00000000..12c05685 --- /dev/null +++ b/client/changes/bug_4435_catch-lock-timeout-exception @@ -0,0 +1 @@ + o Catch lock timeout exception (#4435). diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index d35d3a2a..11e8585b 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -54,6 +54,7 @@ from leap.soledad.common.errors import ( InvalidTokenError, NotLockedError, AlreadyLockedError, + LockTimedOutError, ) from leap.soledad.common.crypto import ( MacMethods, @@ -410,6 +411,8 @@ class Soledad(object): token, timeout = self._shared_db.lock() except AlreadyLockedError: raise BootstrapSequenceError('Database is already locked.') + except LockTimedOutError: + raise BootstrapSequenceError('Lock operation timed out.') try: self._get_or_gen_crypto_secrets() -- cgit v1.2.3 From 360019313b371da17241abbd72038bdebe1b2649 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Jan 2014 13:57:32 -0400 Subject: add rw_lock to client ops --- client/changes/feature_rw_lock_client_ops | 1 + client/src/leap/soledad/client/__init__.py | 80 ++++++++++++++++++------------ 2 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 client/changes/feature_rw_lock_client_ops (limited to 'client') diff --git a/client/changes/feature_rw_lock_client_ops b/client/changes/feature_rw_lock_client_ops new file mode 100644 index 00000000..1c1ab446 --- /dev/null +++ b/client/changes/feature_rw_lock_client_ops @@ -0,0 +1 @@ + o Add a read-write lock for all client operations. Addresses: #4972 diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 11e8585b..48c703ed 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -249,6 +249,7 @@ class Soledad(object): """ syncing_lock = defaultdict(Lock) + rw_lock = Lock() """ A dictionary that hold locks which avoid multiple sync attempts from the same database replica. @@ -790,7 +791,8 @@ class Soledad(object): :rtype: str """ doc.content = self._convert_to_unicode(doc.content) - return self._db.put_doc(doc) + with self.rw_lock: + return self._db.put_doc(doc) def delete_doc(self, doc): """ @@ -802,7 +804,8 @@ class Soledad(object): :return: the new revision identifier for the document :rtype: str """ - return self._db.delete_doc(doc) + with self.rw_lock: + return self._db.delete_doc(doc) def get_doc(self, doc_id, include_deleted=False): """ @@ -818,7 +821,8 @@ class Soledad(object): :return: the document object or None :rtype: SoledadDocument """ - return self._db.get_doc(doc_id, include_deleted=include_deleted) + with self.rw_lock: + return self._db.get_doc(doc_id, include_deleted=include_deleted) def get_docs(self, doc_ids, check_for_conflicts=True, include_deleted=False): @@ -835,9 +839,10 @@ class Soledad(object): in matching doc_ids order. :rtype: generator """ - return self._db.get_docs(doc_ids, - check_for_conflicts=check_for_conflicts, - include_deleted=include_deleted) + with self.rw_lock: + return self._db.get_docs( + doc_ids, check_for_conflicts=check_for_conflicts, + include_deleted=include_deleted) def get_all_docs(self, include_deleted=False): """Get the JSON content for all documents in the database. @@ -849,7 +854,8 @@ class Soledad(object): The current generation of the database, followed by a list of all the documents in the database. """ - return self._db.get_all_docs(include_deleted) + with self.rw_lock: + return self._db.get_all_docs(include_deleted) def _convert_to_unicode(self, content): """ @@ -894,8 +900,9 @@ class Soledad(object): :return: the new document :rtype: SoledadDocument """ - return self._db.create_doc( - self._convert_to_unicode(content), doc_id=doc_id) + with self.rw_lock: + return self._db.create_doc( + self._convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): """ @@ -914,7 +921,8 @@ class Soledad(object): :return: The new cocument :rtype: SoledadDocument """ - return self._db.create_doc_from_json(json, doc_id=doc_id) + with self.rw_lock: + return self._db.create_doc_from_json(json, doc_id=doc_id) def create_index(self, index_name, *index_expressions): """ @@ -938,8 +946,10 @@ class Soledad(object): "number(fieldname, width)", "lower(fieldname)" """ - if self._db: - return self._db.create_index(index_name, *index_expressions) + with self.rw_lock: + if self._db: + return self._db.create_index( + index_name, *index_expressions) def delete_index(self, index_name): """ @@ -948,8 +958,9 @@ class Soledad(object): :param index_name: The name of the index we are removing :type index_name: str """ - if self._db: - return self._db.delete_index(index_name) + with self.rw_lock: + if self._db: + return self._db.delete_index(index_name) def list_indexes(self): """ @@ -958,8 +969,9 @@ class Soledad(object): :return: A list of [('index-name', ['field', 'field2'])] definitions. :rtype: list """ - if self._db: - return self._db.list_indexes() + with self.rw_lock: + if self._db: + return self._db.list_indexes() def get_from_index(self, index_name, *key_values): """ @@ -981,8 +993,9 @@ class Soledad(object): :return: List of [Document] :rtype: list """ - if self._db: - return self._db.get_from_index(index_name, *key_values) + with self.rw_lock: + if self._db: + return self._db.get_from_index(index_name, *key_values) def get_count_from_index(self, index_name, *key_values): """ @@ -998,8 +1011,9 @@ class Soledad(object): :return: count. :rtype: int """ - if self._db: - return self._db.get_count_from_index(index_name, *key_values) + with self.rw_lock: + if self._db: + return self._db.get_count_from_index(index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): """ @@ -1028,9 +1042,10 @@ class Soledad(object): :return: List of [Document] :rtype: list """ - if self._db: - return self._db.get_range_from_index( - index_name, start_value, end_value) + with self.rw_lock: + if self._db: + return self._db.get_range_from_index( + index_name, start_value, end_value) def get_index_keys(self, index_name): """ @@ -1041,8 +1056,9 @@ class Soledad(object): :return: [] A list of tuples of indexed keys. :rtype: list """ - if self._db: - return self._db.get_index_keys(index_name) + with self.rw_lock: + if self._db: + return self._db.get_index_keys(index_name) def get_doc_conflicts(self, doc_id): """ @@ -1054,8 +1070,9 @@ class Soledad(object): :return: a list of the document entries that are conflicted :rtype: list """ - if self._db: - return self._db.get_doc_conflicts(doc_id) + with self.rw_lock: + if self._db: + return self._db.get_doc_conflicts(doc_id) def resolve_doc(self, doc, conflicted_doc_revs): """ @@ -1067,8 +1084,9 @@ class Soledad(object): supersedes. :type conflicted_doc_revs: list """ - if self._db: - return self._db.resolve_doc(doc, conflicted_doc_revs) + with self.rw_lock: + if self._db: + return self._db.resolve_doc(doc, conflicted_doc_revs) def sync(self): """ @@ -1209,7 +1227,7 @@ class Soledad(object): """ soledad_assert(self.STORAGE_SECRETS_KEY in data) # check mac of the recovery document - mac_auth = False + #mac_auth = False # XXX ? mac = None if MAC_KEY in data: soledad_assert(data[MAC_KEY] is not None) @@ -1232,7 +1250,7 @@ class Soledad(object): if mac != data[MAC_KEY]: raise WrongMac('Could not authenticate recovery document\'s ' 'contents.') - mac_auth = True + #mac_auth = True # XXX ? # include secrets in the secret pool. secrets = 0 for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items(): -- cgit v1.2.3 From b72ec5f7229a6371894666bb242799d89a72f36c Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 26 Jan 2014 18:54:22 -0400 Subject: add pragmas for temp_store=mem and synchronous=off controlled by environmental variables --- client/changes/feature_sqlite-optimization-pragmas | 1 + client/src/leap/soledad/client/sqlcipher.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 client/changes/feature_sqlite-optimization-pragmas (limited to 'client') diff --git a/client/changes/feature_sqlite-optimization-pragmas b/client/changes/feature_sqlite-optimization-pragmas new file mode 100644 index 00000000..7a35f005 --- /dev/null +++ b/client/changes/feature_sqlite-optimization-pragmas @@ -0,0 +1 @@ + o Add sync=off and tem_store=mem to soledad client, for optimization. diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 43c871c3..ef059e9b 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -190,6 +190,10 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self._set_crypto_pragmas( self._db_handle, password, raw_key, cipher, kdf_iter, cipher_page_size) + if os.environ.get('LEAP_SQLITE_NOSYNC'): + self._pragma_synchronous_off(self._db_handle) + if os.environ.get('LEAP_SQLITE_MEMSTORE'): + self._pragma_mem_temp_store(self._db_handle) self._real_replica_uid = None self._ensure_schema() self._crypto = crypto @@ -734,6 +738,22 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # XXX change passphrase param! db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % passphrase) + @classmethod + def _pragma_synchronous_off(cls, db_handle): + """ + Change the setting of the "synchronous" flag to OFF. + """ + logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF") + db_handle.cursor().execute('PRAGMA synchronous=OFF') + + @classmethod + def _pragma_mem_temp_store(cls, db_handle): + """ + Use a in-memory store for temporary tables. + """ + logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY") + db_handle.cursor().execute('PRAGMA temp_store=MEMORY') + # Extra query methods: extensions to the base sqlite implmentation. def get_count_from_index(self, index_name, *key_values): -- cgit v1.2.3 From 27a70fbbde42166c268c60e624ed11eac7788b55 Mon Sep 17 00:00:00 2001 From: Ivan Alejandro Date: Mon, 27 Jan 2014 14:49:59 -0300 Subject: Always return unicode, even on UnicodeError. --- client/changes/bug_return-always-unicode | 1 + client/src/leap/soledad/client/__init__.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 client/changes/bug_return-always-unicode (limited to 'client') diff --git a/client/changes/bug_return-always-unicode b/client/changes/bug_return-always-unicode new file mode 100644 index 00000000..f4ee51ed --- /dev/null +++ b/client/changes/bug_return-always-unicode @@ -0,0 +1 @@ + o Always return unicode in helper method, even on UnicodeError. Related to #4998. diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 48c703ed..3fb037c8 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -859,7 +859,7 @@ class Soledad(object): def _convert_to_unicode(self, content): """ - Converts content to utf8 (or all the strings in content) + Converts content to unicode (or all the strings in content) NOTE: Even though this method supports any type, it will currently ignore contents of lists, tuple or any other @@ -874,13 +874,14 @@ class Soledad(object): if isinstance(content, unicode): return content elif isinstance(content, str): + result = chardet.detect(content) + default = "utf-8" + encoding = result["encoding"] or default try: - result = chardet.detect(content) - default = "utf-8" - encoding = result["encoding"] or default content = content.decode(encoding) - except UnicodeError: - pass + except UnicodeError as e: + logger.error("Unicode error: {0!r}. Using 'replace'".format(e)) + content = content.decode(encoding, 'replace') return content else: if isinstance(content, dict): -- cgit v1.2.3 From dfa01cb0518eade316abb12c10bf2dc808745cea Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 11 Feb 2014 14:52:37 -0400 Subject: Remove global client rw lock leap.mail is handling locks in a finer-grained way now, so we do not need to block everything so much --- client/src/leap/soledad/client/__init__.py | 77 ++++++++++++------------------ 1 file changed, 30 insertions(+), 47 deletions(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 3fb037c8..f0abf130 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -249,7 +249,6 @@ class Soledad(object): """ syncing_lock = defaultdict(Lock) - rw_lock = Lock() """ A dictionary that hold locks which avoid multiple sync attempts from the same database replica. @@ -791,8 +790,7 @@ class Soledad(object): :rtype: str """ doc.content = self._convert_to_unicode(doc.content) - with self.rw_lock: - return self._db.put_doc(doc) + return self._db.put_doc(doc) def delete_doc(self, doc): """ @@ -804,8 +802,7 @@ class Soledad(object): :return: the new revision identifier for the document :rtype: str """ - with self.rw_lock: - return self._db.delete_doc(doc) + return self._db.delete_doc(doc) def get_doc(self, doc_id, include_deleted=False): """ @@ -821,8 +818,7 @@ class Soledad(object): :return: the document object or None :rtype: SoledadDocument """ - with self.rw_lock: - return self._db.get_doc(doc_id, include_deleted=include_deleted) + return self._db.get_doc(doc_id, include_deleted=include_deleted) def get_docs(self, doc_ids, check_for_conflicts=True, include_deleted=False): @@ -839,10 +835,9 @@ class Soledad(object): in matching doc_ids order. :rtype: generator """ - with self.rw_lock: - return self._db.get_docs( - doc_ids, check_for_conflicts=check_for_conflicts, - include_deleted=include_deleted) + return self._db.get_docs( + doc_ids, check_for_conflicts=check_for_conflicts, + include_deleted=include_deleted) def get_all_docs(self, include_deleted=False): """Get the JSON content for all documents in the database. @@ -854,8 +849,7 @@ class Soledad(object): The current generation of the database, followed by a list of all the documents in the database. """ - with self.rw_lock: - return self._db.get_all_docs(include_deleted) + return self._db.get_all_docs(include_deleted) def _convert_to_unicode(self, content): """ @@ -901,9 +895,8 @@ class Soledad(object): :return: the new document :rtype: SoledadDocument """ - with self.rw_lock: - return self._db.create_doc( - self._convert_to_unicode(content), doc_id=doc_id) + return self._db.create_doc( + self._convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): """ @@ -922,8 +915,7 @@ class Soledad(object): :return: The new cocument :rtype: SoledadDocument """ - with self.rw_lock: - return self._db.create_doc_from_json(json, doc_id=doc_id) + return self._db.create_doc_from_json(json, doc_id=doc_id) def create_index(self, index_name, *index_expressions): """ @@ -947,10 +939,9 @@ class Soledad(object): "number(fieldname, width)", "lower(fieldname)" """ - with self.rw_lock: - if self._db: - return self._db.create_index( - index_name, *index_expressions) + if self._db: + return self._db.create_index( + index_name, *index_expressions) def delete_index(self, index_name): """ @@ -959,9 +950,8 @@ class Soledad(object): :param index_name: The name of the index we are removing :type index_name: str """ - with self.rw_lock: - if self._db: - return self._db.delete_index(index_name) + if self._db: + return self._db.delete_index(index_name) def list_indexes(self): """ @@ -970,9 +960,8 @@ class Soledad(object): :return: A list of [('index-name', ['field', 'field2'])] definitions. :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.list_indexes() + if self._db: + return self._db.list_indexes() def get_from_index(self, index_name, *key_values): """ @@ -994,9 +983,8 @@ class Soledad(object): :return: List of [Document] :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.get_from_index(index_name, *key_values) + if self._db: + return self._db.get_from_index(index_name, *key_values) def get_count_from_index(self, index_name, *key_values): """ @@ -1012,9 +1000,8 @@ class Soledad(object): :return: count. :rtype: int """ - with self.rw_lock: - if self._db: - return self._db.get_count_from_index(index_name, *key_values) + if self._db: + return self._db.get_count_from_index(index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): """ @@ -1043,10 +1030,9 @@ class Soledad(object): :return: List of [Document] :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.get_range_from_index( - index_name, start_value, end_value) + if self._db: + return self._db.get_range_from_index( + index_name, start_value, end_value) def get_index_keys(self, index_name): """ @@ -1057,9 +1043,8 @@ class Soledad(object): :return: [] A list of tuples of indexed keys. :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.get_index_keys(index_name) + if self._db: + return self._db.get_index_keys(index_name) def get_doc_conflicts(self, doc_id): """ @@ -1071,9 +1056,8 @@ class Soledad(object): :return: a list of the document entries that are conflicted :rtype: list """ - with self.rw_lock: - if self._db: - return self._db.get_doc_conflicts(doc_id) + if self._db: + return self._db.get_doc_conflicts(doc_id) def resolve_doc(self, doc, conflicted_doc_revs): """ @@ -1085,9 +1069,8 @@ class Soledad(object): supersedes. :type conflicted_doc_revs: list """ - with self.rw_lock: - if self._db: - return self._db.resolve_doc(doc, conflicted_doc_revs) + if self._db: + return self._db.resolve_doc(doc, conflicted_doc_revs) def sync(self): """ -- cgit v1.2.3 From 057e7c28f9fc790fa449cef5361fba9dcd5009d1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 19 Feb 2014 23:34:49 -0400 Subject: add locks for create_doc and update_indexes. Closes: #5139 This solves a InterfaceError (sqlite error code 21) we were having with massive concurrent creation/puts. --- client/src/leap/soledad/client/sqlcipher.py | 31 +++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index ef059e9b..d8ba0b79 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -147,6 +147,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): _index_storage_value = 'expand referenced encrypted' k_lock = threading.Lock() + create_doc_lock = threading.Lock() + update_indexes_lock = threading.Lock() _syncer = None def __init__(self, sqlcipher_file, password, document_factory=None, @@ -400,6 +402,22 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') + def create_doc(self, content, doc_id=None): + """ + Create a new document in the local encrypted database. + + :param content: the contents of the new document + :type content: dict + :param doc_id: an optional identifier specifying the document id + :type doc_id: str + + :return: the new document + :rtype: SoledadDocument + """ + with self.create_doc_lock: + return sqlite_backend.SQLitePartialExpandDatabase.create_doc( + self, content, doc_id=doc_id) + def _put_and_update_indexes(self, old_doc, doc): """ Update a document and all indexes related to it. @@ -409,12 +427,13 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :param doc: The new version of the document. :type doc: u1db.Document """ - sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( - self, old_doc, doc) - c = self._db_handle.cursor() - c.execute('UPDATE document SET syncable=? ' - 'WHERE doc_id=?', - (doc.syncable, doc.doc_id)) + with self.update_indexes_lock: + sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( + self, old_doc, doc) + c = self._db_handle.cursor() + c.execute('UPDATE document SET syncable=? ' + 'WHERE doc_id=?', + (doc.syncable, doc.doc_id)) def _get_doc(self, doc_id, check_for_conflicts=False): """ -- cgit v1.2.3 From 2404b07cc015c4ce76425b7bcf1277a6bbfded64 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 19 Feb 2014 23:50:23 -0400 Subject: Set Write-Ahead Logging with autocommit set to 50 pages, a value that will permit fast reads. also set synchronous mode to normal on regular operation. --- client/src/leap/soledad/client/sqlcipher.py | 45 +++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) (limited to 'client') diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index d8ba0b79..09efa592 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -194,8 +194,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): cipher_page_size) if os.environ.get('LEAP_SQLITE_NOSYNC'): self._pragma_synchronous_off(self._db_handle) + else: + self._pragma_synchronous_normal(self._db_handle) if os.environ.get('LEAP_SQLITE_MEMSTORE'): self._pragma_mem_temp_store(self._db_handle) + self._pragma_write_ahead_logging(self._db_handle) self._real_replica_uid = None self._ensure_schema() self._crypto = crypto @@ -765,6 +768,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF") db_handle.cursor().execute('PRAGMA synchronous=OFF') + @classmethod + def _pragma_synchronous_normal(cls, db_handle): + """ + Change the setting of the "synchronous" flag to NORMAL. + """ + logger.debug("SQLCIPHER: SETTING SYNCHRONOUS NORMAL") + db_handle.cursor().execute('PRAGMA synchronous=NORMAL') + @classmethod def _pragma_mem_temp_store(cls, db_handle): """ @@ -773,6 +784,40 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY") db_handle.cursor().execute('PRAGMA temp_store=MEMORY') + @classmethod + def _pragma_write_ahead_logging(cls, db_handle): + """ + Enable write-ahead logging, and set the autocheckpoint to 50 pages. + + Setting the autocheckpoint to a small value, we make the reads not + suffer too much performance degradation. + + From the sqlite docs: + + "There is a tradeoff between average read performance and average write + performance. To maximize the read performance, one wants to keep the + WAL as small as possible and hence run checkpoints frequently, perhaps + as often as every COMMIT. To maximize write performance, one wants to + amortize the cost of each checkpoint over as many writes as possible, + meaning that one wants to run checkpoints infrequently and let the WAL + grow as large as possible before each checkpoint. The decision of how + often to run checkpoints may therefore vary from one application to + another depending on the relative read and write performance + requirements of the application. The default strategy is to run a + checkpoint once the WAL reaches 1000 pages" + """ + logger.debug("SQLCIPHER: SETTING WRITE-AHEAD LOGGING") + db_handle.cursor().execute('PRAGMA journal_mode=WAL') + # The optimum value can still use a little bit of tuning, but we favor + # small sizes of the WAL file to get fast reads, since we assume that + # the writes will be quick enough to not block too much. + + # TODO + # As a further improvement, we might want to set autocheckpoint to 0 + # here and do the checkpoints manually in a separate thread, to avoid + # any blocks in the main thread (we should run a loopingcall from here) + db_handle.cursor().execute('PRAGMA wal_autocheckpoint=50') + # Extra query methods: extensions to the base sqlite implmentation. def get_count_from_index(self, index_name, *key_values): -- cgit v1.2.3 From c2e4bad20fa17b92591d861dff2f20ca71610319 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 19 Feb 2014 23:56:27 -0400 Subject: changes file --- client/changes/bug_5139-interface-error | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 client/changes/bug_5139-interface-error (limited to 'client') diff --git a/client/changes/bug_5139-interface-error b/client/changes/bug_5139-interface-error new file mode 100644 index 00000000..9127e70b --- /dev/null +++ b/client/changes/bug_5139-interface-error @@ -0,0 +1,2 @@ +o Add lock for create_doc and update_indexes call, + prevents concurrent access to the db. Closes #5139. -- cgit v1.2.3 From dcff2ecd5ca63b09f5e40cca4e18c4660406d5d6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 28 Feb 2014 05:43:27 +0900 Subject: backward-compatibility for socket.create_connection in 2.6 Closes: #5208 --- client/changes/bug_5208_support_socket_26 | 1 + client/src/leap/soledad/client/__init__.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 client/changes/bug_5208_support_socket_26 (limited to 'client') diff --git a/client/changes/bug_5208_support_socket_26 b/client/changes/bug_5208_support_socket_26 new file mode 100644 index 00000000..b44d1d4e --- /dev/null +++ b/client/changes/bug_5208_support_socket_26 @@ -0,0 +1 @@ + o Back-compatibility for socket.create_connection interface in 2.6. Closes: #5208 diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index f0abf130..b5ce7c32 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -1311,9 +1311,17 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection): # derived from httplib.py def connect(self): - "Connect to a host on a given (SSL) port." - sock = socket.create_connection((self.host, self.port), - SOLEDAD_TIMEOUT, self.source_address) + """ + Connect to a host on a given (SSL) port. + """ + try: + source = self.source_address + sock = socket.create_connection((self.host, self.port), + SOLEDAD_TIMEOUT, source) + except AttributeError: + # source_address was introduced in 2.7 + sock = socket.create_connection((self.host, self.port), + SOLEDAD_TIMEOUT) if self._tunnel_host: self.sock = sock self._tunnel() -- cgit v1.2.3 From b479b64691fef819996df0d62630a0fe0b1bb1a8 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 10 Mar 2014 03:41:17 -0400 Subject: minor docstring corrections --- client/src/leap/soledad/client/sqlcipher.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 09efa592..3aea340d 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -91,10 +91,10 @@ def open(path, password, create=True, document_factory=None, crypto=None, database does not already exist. :param path: The filesystem path for the database to open. - :param type: str + :type path: str :param create: True/False, should the database be created if it doesn't already exist? - :param type: bool + :param create: bool :param document_factory: A function that will be called with the same parameters as Document.__init__. :type document_factory: callable @@ -155,20 +155,22 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024): """ - Create a new sqlcipher file. + Connect to an existing SQLCipher database, creating a new sqlcipher + database file if needed. :param sqlcipher_file: The path for the SQLCipher file. :type sqlcipher_file: str :param password: The password that protects the SQLCipher db. :type password: str :param document_factory: A function that will be called with the same - parameters as Document.__init__. + parameters as Document.__init__. :type document_factory: callable :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. + document contents when syncing. :type crypto: soledad.crypto.SoledadCrypto - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. + :param raw_key: Whether password is a raw 64-char hex string or a + passphrase that should be hashed to obtain the + encyrption key. :type raw_key: bool :param cipher: The cipher and mode to use. :type cipher: str -- cgit v1.2.3 From e5664fb1046e71589d0c41cd605761cc642ff28b Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 10 Mar 2014 15:11:02 -0300 Subject: Fix docstring typo. --- client/src/leap/soledad/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'client') diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index b5ce7c32..a8d68c88 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -778,7 +778,7 @@ class Soledad(object): ============================== WARNING ============================== This method converts the document's contents to unicode in-place. This - meanse that after calling C{put_doc(doc)}, the contents of the + means that after calling C{put_doc(doc)}, the contents of the document, i.e. C{doc.content}, might be different from before the call. ============================== WARNING ============================== -- cgit v1.2.3 From 7fda4c5448cd3566a802777149c511a1ccc2a143 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 12 Mar 2014 17:27:42 -0300 Subject: Do not autocreate remote db when syncing (#5302). --- client/changes/feature_5302_do-not-create-user-db-when-syncing | 2 ++ client/src/leap/soledad/client/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 client/changes/feature_5302_do-not-create-user-db-when-syncing (limited to 'client') diff --git a/client/changes/feature_5302_do-not-create-user-db-when-syncing b/client/changes/feature_5302_do-not-create-user-db-when-syncing new file mode 100644 index 00000000..c3270ab1 --- /dev/null +++ b/client/changes/feature_5302_do-not-create-user-db-when-syncing @@ -0,0 +1,2 @@ + o Do not autocreate remote user database when syncing. Tapicero should make + sure that that db is created when the user is created. Closes #5302. diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index a8d68c88..46e3cd5f 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -1091,7 +1091,7 @@ class Soledad(object): with Soledad.syncing_lock[self._db._get_replica_uid()]: local_gen = self._db.sync( urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=True) + creds=self._creds, autocreate=False) signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) return local_gen -- cgit v1.2.3 From a3fed4d42ab4a7be7bc7ebe86b35805ac73d62de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Touceda?= Date: Fri, 4 Apr 2014 16:34:33 -0300 Subject: Fold in changes --- client/changes/bug_4435_catch-lock-timeout-exception | 1 - client/changes/bug_5139-interface-error | 2 -- client/changes/bug_5208_support_socket_26 | 1 - client/changes/bug_return-always-unicode | 1 - client/changes/bug_reuse-http-connection | 2 -- client/changes/bug_unlock_shared_if_fails | 2 -- client/changes/feature_4451_avoid_concurrent_syncs | 2 -- client/changes/feature_4616_sqlite_count_by_index | 1 - client/changes/feature_5302_do-not-create-user-db-when-syncing | 2 -- client/changes/feature_rw_lock_client_ops | 1 - client/changes/feature_sqlite-optimization-pragmas | 1 - 11 files changed, 16 deletions(-) delete mode 100644 client/changes/bug_4435_catch-lock-timeout-exception delete mode 100644 client/changes/bug_5139-interface-error delete mode 100644 client/changes/bug_5208_support_socket_26 delete mode 100644 client/changes/bug_return-always-unicode delete mode 100644 client/changes/bug_reuse-http-connection delete mode 100644 client/changes/bug_unlock_shared_if_fails delete mode 100644 client/changes/feature_4451_avoid_concurrent_syncs delete mode 100644 client/changes/feature_4616_sqlite_count_by_index delete mode 100644 client/changes/feature_5302_do-not-create-user-db-when-syncing delete mode 100644 client/changes/feature_rw_lock_client_ops delete mode 100644 client/changes/feature_sqlite-optimization-pragmas (limited to 'client') diff --git a/client/changes/bug_4435_catch-lock-timeout-exception b/client/changes/bug_4435_catch-lock-timeout-exception deleted file mode 100644 index 12c05685..00000000 --- a/client/changes/bug_4435_catch-lock-timeout-exception +++ /dev/null @@ -1 +0,0 @@ - o Catch lock timeout exception (#4435). diff --git a/client/changes/bug_5139-interface-error b/client/changes/bug_5139-interface-error deleted file mode 100644 index 9127e70b..00000000 --- a/client/changes/bug_5139-interface-error +++ /dev/null @@ -1,2 +0,0 @@ -o Add lock for create_doc and update_indexes call, - prevents concurrent access to the db. Closes #5139. diff --git a/client/changes/bug_5208_support_socket_26 b/client/changes/bug_5208_support_socket_26 deleted file mode 100644 index b44d1d4e..00000000 --- a/client/changes/bug_5208_support_socket_26 +++ /dev/null @@ -1 +0,0 @@ - o Back-compatibility for socket.create_connection interface in 2.6. Closes: #5208 diff --git a/client/changes/bug_return-always-unicode b/client/changes/bug_return-always-unicode deleted file mode 100644 index f4ee51ed..00000000 --- a/client/changes/bug_return-always-unicode +++ /dev/null @@ -1 +0,0 @@ - o Always return unicode in helper method, even on UnicodeError. Related to #4998. diff --git a/client/changes/bug_reuse-http-connection b/client/changes/bug_reuse-http-connection deleted file mode 100644 index c6cdd9b4..00000000 --- a/client/changes/bug_reuse-http-connection +++ /dev/null @@ -1,2 +0,0 @@ - o Fix a bug in soledad.client.sqlcipher by which we were creating - a new connection for each sync. diff --git a/client/changes/bug_unlock_shared_if_fails b/client/changes/bug_unlock_shared_if_fails deleted file mode 100644 index fc5716e4..00000000 --- a/client/changes/bug_unlock_shared_if_fails +++ /dev/null @@ -1,2 +0,0 @@ - o Unlock shared_db if anything fails in the bootstrap - sequence. Fixes #4702. \ No newline at end of file diff --git a/client/changes/feature_4451_avoid_concurrent_syncs b/client/changes/feature_4451_avoid_concurrent_syncs deleted file mode 100644 index 04a2c4df..00000000 --- a/client/changes/feature_4451_avoid_concurrent_syncs +++ /dev/null @@ -1,2 +0,0 @@ - o Avoid concurrent syncs for the same account, but allow for distinct - accounts (4451). diff --git a/client/changes/feature_4616_sqlite_count_by_index b/client/changes/feature_4616_sqlite_count_by_index deleted file mode 100644 index c7819d38..00000000 --- a/client/changes/feature_4616_sqlite_count_by_index +++ /dev/null @@ -1 +0,0 @@ - o Adds a get_count_by_index to sqlcipher u1db backend. Related to: #4616 diff --git a/client/changes/feature_5302_do-not-create-user-db-when-syncing b/client/changes/feature_5302_do-not-create-user-db-when-syncing deleted file mode 100644 index c3270ab1..00000000 --- a/client/changes/feature_5302_do-not-create-user-db-when-syncing +++ /dev/null @@ -1,2 +0,0 @@ - o Do not autocreate remote user database when syncing. Tapicero should make - sure that that db is created when the user is created. Closes #5302. diff --git a/client/changes/feature_rw_lock_client_ops b/client/changes/feature_rw_lock_client_ops deleted file mode 100644 index 1c1ab446..00000000 --- a/client/changes/feature_rw_lock_client_ops +++ /dev/null @@ -1 +0,0 @@ - o Add a read-write lock for all client operations. Addresses: #4972 diff --git a/client/changes/feature_sqlite-optimization-pragmas b/client/changes/feature_sqlite-optimization-pragmas deleted file mode 100644 index 7a35f005..00000000 --- a/client/changes/feature_sqlite-optimization-pragmas +++ /dev/null @@ -1 +0,0 @@ - o Add sync=off and tem_store=mem to soledad client, for optimization. -- cgit v1.2.3