From 9f7285403f6e012b8ea5406cf9a2a19e68412210 Mon Sep 17 00:00:00 2001 From: varac Date: Tue, 29 Sep 2015 15:39:25 +0200 Subject: [docs] added couchdb link to README --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index a2755f92..36746c82 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,9 @@ -Soledad +Soledad ================================================================== *Synchronization Of Locally Encrypted Data Among Devices* Soledad is the part of LEAP that allows application data to be -securely shared among devices. It provides, to other parts of the +securely shared among devices. It provides, to other parts of the LEAP project, an API for data storage and sync. This software is under development. @@ -53,7 +53,7 @@ to run tests in development mode you must do the following:: scripts/develop_mode.sh ./run_tests.sh -Note that to run CouchDB tests, be sure you have ``CouchDB`` installed on your +Note that to run CouchDB tests, be sure you have [CouchDB](https://couchdb.apache.org/) installed on your system. -- cgit v1.2.3 From 4964eed77748d65df30847a5fe0a608a163af7e0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 30 Sep 2015 21:35:05 -0400 Subject: [docs] fix rst link markup --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 36746c82..631f90bb 100644 --- a/README.rst +++ b/README.rst @@ -53,9 +53,10 @@ to run tests in development mode you must do the following:: scripts/develop_mode.sh ./run_tests.sh -Note that to run CouchDB tests, be sure you have [CouchDB](https://couchdb.apache.org/) installed on your +Note that to run CouchDB tests, be sure you have `CouchDB`_ installed on your system. +.. _`CouchDB`: https://couchdb.apache.org/ Privileges ----- -- cgit v1.2.3 From 7a0dba8b7008aca8652f7b334352d9ca63cb2054 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 30 Sep 2015 15:27:21 -0300 Subject: [feat] read netrc path from configuration file netrc file was hardcoded inside create-user-db. Now it reads the path from /etc/leap/soledad-server.conf as done on server process. The new configuration property is called 'admin_netrc'. --- server/changes/create_db_cmd | 1 + server/pkg/create-user-db | 3 ++- server/src/leap/soledad/server/__init__.py | 11 ++++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/server/changes/create_db_cmd b/server/changes/create_db_cmd index cee0a935..964a7906 100644 --- a/server/changes/create_db_cmd +++ b/server/changes/create_db_cmd @@ -1,3 +1,4 @@ o Adds a new config parameter 'create_cmd', which allows sysadmin to specify which command will create a database. That command was added in pkg/create-user-db and debian package automates steps needed for sudo access. + o Read netrc path from configuration file for create-user-db command. diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index 1a7e77a7..ae2f0f57 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -21,6 +21,7 @@ import netrc import argparse from leap.soledad.common.couch import CouchDatabase from leap.soledad.common.couch import is_db_name_valid +from leap.soledad.server import load_configuration description = """ @@ -30,7 +31,7 @@ This is meant to be used by Soledad Server. parser = argparse.ArgumentParser(description=description) parser.add_argument('dbname', metavar='user-d34db33f', type=str, help='database name on the format user-{uuid4}') -NETRC_PATH = '/etc/couchdb/couchdb-admin.netrc' +NETRC_PATH = load_configuration('/etc/leap/soledad-server.conf')['admin_netrc'] def url_for_db(dbname): diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index bb1c6db0..1273d15c 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -283,19 +283,20 @@ def load_configuration(file_path): @return: A dictionary with the configuration. @rtype: dict """ - conf = { + defaults = { 'couch_url': 'http://localhost:5984', - 'create_cmd': None + 'create_cmd': None, + 'admin_netrc': '/etc/couchdb/couchdb-admin.netrc', } config = configparser.ConfigParser() config.read(file_path) if 'soledad-server' in config: - for key in conf: + for key in defaults: if key in config['soledad-server']: - conf[key] = config['soledad-server'][key] + defaults[key] = config['soledad-server'][key] # TODO: implement basic parsing/sanitization of options comming from # config file. - return conf + return defaults # ---------------------------------------------------------------------------- -- cgit v1.2.3 From 234e5879188fe100bf99b9ef6f252abff06b4dad Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 2 Oct 2015 12:03:29 -0400 Subject: [bug] do not signal sync completion if failed - Related: #7503 --- client/changes/bug_7503-do-not-signal-sync-complete | 1 + client/src/leap/soledad/client/api.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 client/changes/bug_7503-do-not-signal-sync-complete diff --git a/client/changes/bug_7503-do-not-signal-sync-complete b/client/changes/bug_7503-do-not-signal-sync-complete new file mode 100644 index 00000000..4cc361e0 --- /dev/null +++ b/client/changes/bug_7503-do-not-signal-sync-complete @@ -0,0 +1 @@ +o Do not signal sync completion if sync failed. Closes: #7503 diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index a558addd..8c5f7f1b 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -723,7 +723,7 @@ class Soledad(object): return passthrough d.addCallbacks(_sync_callback, _sync_errback) - d.addBoth(_emit_done_data_sync) + d.addCallback(_emit_done_data_sync) return d @property -- cgit v1.2.3 From 2121d21945ef7458eb57c3cab53a98b92fce1417 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 2 Oct 2015 11:54:54 -0400 Subject: [bug] increase http request timeout time to 90s this is a workaroud to reduce the chances of failed sync due to timeouts. this should be properly tackled by: 1. implementing proper cancellable for the sync operation. 2. implementing a retry count at the level of a single request, handled internally by soledad. in this way we can remove the retries logic from the soledadbootstrapper in the bitmask client. - Related: #7382 --- client/src/leap/soledad/client/http_target/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py index 7a5cea9f..498fb6e7 100644 --- a/client/src/leap/soledad/client/http_target/__init__.py +++ b/client/src/leap/soledad/client/http_target/__init__.py @@ -87,4 +87,8 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): # asynchronous encryption/decryption attributes self._decryption_callback = None self._sync_decr_pool = None - self._http = HTTPClient(cert_file) + + # XXX Increasing timeout of simple requests to avoid chances of hitting + # the duplicated syncing bug. This could be reduced to the 30s default + # after implementing Cancellable Sync. See #7382 + self._http = HTTPClient(cert_file, timeout=90) -- cgit v1.2.3 From a48b630ef48377b7307b59333f8eb02190786012 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 2 Oct 2015 17:37:52 -0300 Subject: [pkg] pin beaker and couchdb for wheezy Wheezy has python-couchdb 0.8 and python-beaker 1.6.3. Pinning them to avoid false positives on tests. --- server/pkg/requirements.pip | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index 3e1aa992..58834d0e 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -1,10 +1,11 @@ configparser -couchdb u1db routes PyOpenSSL twisted -Beaker +#pinned for wheezy compatibility +Beaker==1.6.3 #wheezy +couchdb==0.8 #wheezy # XXX -- fix me! # oauth is not strictly needed by us, but we need it until u1db adds it to its -- cgit v1.2.3 From c7baad4f07ec8a44bd12113775c40974be3a4fb9 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 2 Oct 2015 17:44:11 -0300 Subject: [bug] Migrate back to python-couchdb 0.8 Wheezy is still at 0.8 and it is yet supported. This commit changes all necessary calls from python-couchdb 1.0 back to python-couchdb 0.8. We can migrate this back to simpler implementation with python-couchdb 1.0 when support for wheezy is dropped. --- common/src/leap/soledad/common/couch.py | 20 ++++++++++++++++---- common/src/leap/soledad/common/tests/test_couch.py | 4 ++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 4c5f6400..10a1b277 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -461,7 +461,8 @@ class CouchDatabase(CommonBackend): """ for ddoc_name in ['docs', 'syncs', 'transactions']: try: - self._database.info(ddoc_name) + self._database.resource('_design', + ddoc_name, '_info').get_json() except ResourceNotFound: ddoc = json.loads( binascii.a2b_base64( @@ -478,10 +479,10 @@ class CouchDatabase(CommonBackend): This is achieved by creating a _security design document, see: http://docs.couchdb.org/en/latest/api/database/security.html """ - security = self._database.security + security = self._database.resource.get_json('_security')[2] security['members'] = {'names': ['soledad'], 'roles': []} security['admins'] = {'names': [], 'roles': []} - self._database.security = security + self._database.resource.put_json('_security', body=security) def get_sync_target(self): """ @@ -888,7 +889,7 @@ class CouchDatabase(CommonBackend): try: resource = self._new_resource() resource.put_json( - doc.doc_id, body=buf.getvalue(), headers=envelope.headers) + doc.doc_id, body=str(buf.getvalue()), headers=envelope.headers) except ResourceConflict: raise RevisionConflict() if self.replica_uid + '_gen' in self.cache: @@ -1337,6 +1338,17 @@ class CouchDatabase(CommonBackend): in matching doc_ids order. :rtype: iterable """ + # Workaround for: + # + # http://bugs.python.org/issue7980 + # https://leap.se/code/issues/5449 + # + # python-couchdb uses time.strptime, which is not thread safe. In + # order to avoid the problem described on the issues above, we preload + # strptime here by evaluating the conversion of an arbitrary date. + # This will not be needed when/if we switch from python-couchdb to + # paisley. + time.strptime('Mar 8 1917', '%b %d %Y') get_one = lambda doc_id: self._get_doc(doc_id, check_for_conflicts) docs = [THREAD_POOL.apply_async(get_one, [doc_id]) for doc_id in doc_ids] diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index d1a07a3a..b4797f5e 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -1507,9 +1507,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): will have the lowest privileged access to an user db. """ self.create_db(ensure=False) - self.assertFalse(self.db._database.security) + self.assertFalse(self.db._database.resource.get_json('_security')[2]) self.db.ensure_security_ddoc() - security_ddoc = self.db._database.security + security_ddoc = self.db._database.resource.get_json('_security')[2] self.assertIn('admins', security_ddoc) self.assertFalse(security_ddoc['admins']['names']) self.assertIn('members', security_ddoc) -- cgit v1.2.3 From 147986547f400f79e8bc0d50af5ae7b5d2a140b4 Mon Sep 17 00:00:00 2001 From: varac Date: Mon, 5 Oct 2015 11:54:56 +0200 Subject: [feat] Move config dir to /etc/soledad - Resolves: #7509 --- README.rst | 2 +- scripts/db_access/reset_db.py | 2 +- scripts/db_access/server_side_db.py | 4 ++-- scripts/ddocs/update_design_docs.py | 2 +- scripts/profiling/doc_put_memory_usage/find_max_upload_size.py | 2 +- server/changes/change_soledad_configdir | 2 ++ server/pkg/create-user-db | 2 +- server/pkg/soledad-server | 8 ++++---- server/src/leap/soledad/server/__init__.py | 2 +- 9 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 server/changes/change_soledad_configdir diff --git a/README.rst b/README.rst index 631f90bb..815db808 100644 --- a/README.rst +++ b/README.rst @@ -79,7 +79,7 @@ The debian package will do the following in order to automate this: * grant restricted sudo access, that only enables user ``soledad`` to call this exact command via ``soledad-admin`` user. -The server side process, configured via ``/etc/leap/soledad-server.conf``, will +The server side process, configured via ``/etc/soledad/soledad-server.conf``, will then use a parameter called 'create_cmd' to know which command is used to allocate new databases. All steps of creation process is then handled automatically by the server, following the same logic as u1db server. diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py index 7c6d281b..c48b511e 100644 --- a/scripts/db_access/reset_db.py +++ b/scripts/db_access/reset_db.py @@ -63,7 +63,7 @@ def get_url(empty): if empty is False: # get couch url cp = ConfigParser() - cp.read('/etc/leap/soledad-server.conf') + cp.read('/etc/soledad/soledad-server.conf') url = cp.get('soledad-server', 'couch_url') else: with open('/etc/couchdb/couchdb.netrc') as f: diff --git a/scripts/db_access/server_side_db.py b/scripts/db_access/server_side_db.py index 18641a0f..fcdd14b6 100644 --- a/scripts/db_access/server_side_db.py +++ b/scripts/db_access/server_side_db.py @@ -1,7 +1,7 @@ #!/usr/bin/python # This script gives server-side access to one Soledad user database by using -# the configuration stored in /etc/leap/soledad-server.conf. +# the configuration stored in /etc/soledad/soledad-server.conf. # # Use it like this: # @@ -20,7 +20,7 @@ uuid = sys.argv[1] # get couch url cp = ConfigParser() -cp.read('/etc/leap/soledad-server.conf') +cp.read('/etc/soledad/soledad-server.conf') url = cp.get('soledad-server', 'couch_url') # access user db diff --git a/scripts/ddocs/update_design_docs.py b/scripts/ddocs/update_design_docs.py index 2e2fa8f0..281482b8 100644 --- a/scripts/ddocs/update_design_docs.py +++ b/scripts/ddocs/update_design_docs.py @@ -50,7 +50,7 @@ def _parse_args(): def _get_url(): # get couch url cp = ConfigParser() - cp.read('/etc/leap/soledad-server.conf') + cp.read('/etc/soledad/soledad-server.conf') url = urlparse(cp.get('soledad-server', 'couch_url')) # get admin password netloc = re.sub('^.*@', '', url.netloc) diff --git a/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py index 02c68015..1a603fd0 100755 --- a/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py +++ b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py @@ -30,7 +30,7 @@ from socket import error as socket_error from leap.soledad.common.couch import CouchDatabase -SOLEDAD_CONFIG_FILE = '/etc/leap/soledad-server.conf' +SOLEDAD_CONFIG_FILE = '/etc/soledad/soledad-server.conf' PREFIX = '/tmp/soledad_test' LOG_FORMAT = '%(asctime)s %(levelname)s %(message)s' RETRIES = 3 # number of times to retry uploading a document of a certain diff --git a/server/changes/change_soledad_configdir b/server/changes/change_soledad_configdir new file mode 100644 index 00000000..710b9ac8 --- /dev/null +++ b/server/changes/change_soledad_configdir @@ -0,0 +1,2 @@ +o Moves config directory from /etc/leap to /etc/soledad + resolves #7509 diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index ae2f0f57..7eafc945 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -31,7 +31,7 @@ This is meant to be used by Soledad Server. parser = argparse.ArgumentParser(description=description) parser.add_argument('dbname', metavar='user-d34db33f', type=str, help='database name on the format user-{uuid4}') -NETRC_PATH = load_configuration('/etc/leap/soledad-server.conf')['admin_netrc'] +NETRC_PATH = load_configuration('/etc/soledad/soledad-server.conf')['admin_netrc'] def url_for_db(dbname): diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server index 811ad55b..74ed122e 100644 --- a/server/pkg/soledad-server +++ b/server/pkg/soledad-server @@ -11,12 +11,12 @@ PATH=/sbin:/bin:/usr/sbin:/usr/bin PIDFILE=/var/run/soledad.pid -RUNDIR=/var/lib/soledad/ OBJ=leap.soledad.server.application LOGFILE=/var/log/soledad.log HTTPS_PORT=2424 -CERT_PATH=/etc/leap/soledad-server.pem -PRIVKEY_PATH=/etc/leap/soledad-server.key +CONFDIR=/etc/soledad +CERT_PATH="${CONFDIR}/soledad-server.pem" +PRIVKEY_PATH="${CONFDIR}/soledad-server.key" TWISTD_PATH=/usr/bin/twistd HOME=/var/lib/soledad/ SSL_METHOD=SSLv23_METHOD @@ -25,7 +25,7 @@ GROUP=soledad [ -r /etc/default/soledad ] && . /etc/default/soledad -test -r /etc/leap/ || exit 0 +test -r ${CONFDIR} || exit 0 . /lib/lsb/init-functions diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 1273d15c..f64d07bf 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -304,7 +304,7 @@ def load_configuration(file_path): # ---------------------------------------------------------------------------- def application(environ, start_response): - conf = load_configuration('/etc/leap/soledad-server.conf') + conf = load_configuration('/etc/soledad/soledad-server.conf') state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) # WSGI application that may be used by `twistd -web` application = GzipMiddleware( -- cgit v1.2.3 From dccaf79789b5f52848cfedd82ca4a2999de1432c Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 5 Oct 2015 12:19:12 -0300 Subject: [bug] fix leap mx by default false on ensure_ddocs ensure_ddocs is a privileged operation. The code was defaulting to True, which caused unprivileged code to fail. This commit changes it to False, forcing you to check your privileges and declare a new argument when calling in order to ensure that this behavior is only supposed to happen on privileged parts. --- common/src/leap/soledad/common/couch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 10a1b277..3dee1473 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -404,7 +404,7 @@ class CouchDatabase(CommonBackend): return cls( url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs) - def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True): + def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=False): """ Create a new Couch data container. -- cgit v1.2.3