Merge branch 'develop' into debian/platform-0.8

16 files changed, 55 insertions, 31 deletions

diff --git a/README.rst b/README.rst
index a2755f92..815db808 100644
--- a/README.rst
+++ b/README.rst
@@ -1,9 +1,9 @@
-Soledad 
+Soledad
 ==================================================================
 *Synchronization Of Locally Encrypted Data Among Devices*
 
 Soledad is the part of LEAP that allows application data to be
-securely shared among devices. It provides, to other parts of the 
+securely shared among devices. It provides, to other parts of the
 LEAP project, an API for data storage and sync.
 
 This software is under development.
@@ -53,9 +53,10 @@ to run tests in development mode you must do the following::
   scripts/develop_mode.sh
   ./run_tests.sh
 
-Note that to run CouchDB tests, be sure you have ``CouchDB`` installed on your
+Note that to run CouchDB tests, be sure you have `CouchDB`_ installed on your
 system.
 
+.. _`CouchDB`: https://couchdb.apache.org/
 
 Privileges
 -----
@@ -78,7 +79,7 @@ The debian package will do the following in order to automate this:
 * grant restricted sudo access, that only enables user ``soledad`` to call this
   exact command via ``soledad-admin`` user.
 
-The server side process, configured via ``/etc/leap/soledad-server.conf``, will
+The server side process, configured via ``/etc/soledad/soledad-server.conf``, will
 then use a parameter called 'create_cmd' to know which command is used to
 allocate new databases. All steps of creation process is then handled
 automatically by the server, following the same logic as u1db server.
diff --git a/client/changes/bug_7503-do-not-signal-sync-complete b/client/changes/bug_7503-do-not-signal-sync-complete
new file mode 100644
index 00000000..4cc361e0
--- /dev/null
+++ b/client/changes/bug_7503-do-not-signal-sync-complete
@@ -0,0 +1 @@
+o Do not signal sync completion if sync failed. Closes: #7503
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index a558addd..8c5f7f1b 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -723,7 +723,7 @@ class Soledad(object):
             return passthrough
 
         d.addCallbacks(_sync_callback, _sync_errback)
-        d.addBoth(_emit_done_data_sync)
+        d.addCallback(_emit_done_data_sync)
         return d
 
     @property
diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py
index 7a5cea9f..498fb6e7 100644
--- a/client/src/leap/soledad/client/http_target/__init__.py
+++ b/client/src/leap/soledad/client/http_target/__init__.py
@@ -87,4 +87,8 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher):
         # asynchronous encryption/decryption attributes
         self._decryption_callback = None
         self._sync_decr_pool = None
-        self._http = HTTPClient(cert_file)
+
+        # XXX Increasing timeout of simple requests to avoid chances of hitting
+        # the duplicated syncing bug. This could be reduced to the 30s default
+        # after implementing Cancellable Sync. See #7382
+        self._http = HTTPClient(cert_file, timeout=90)
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 4c5f6400..3dee1473 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -404,7 +404,7 @@ class CouchDatabase(CommonBackend):
         return cls(
             url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs)
 
-    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True):
+    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=False):
         """
         Create a new Couch data container.
 
@@ -461,7 +461,8 @@ class CouchDatabase(CommonBackend):
         """
         for ddoc_name in ['docs', 'syncs', 'transactions']:
             try:
-                self._database.info(ddoc_name)
+                self._database.resource('_design',
+                                        ddoc_name, '_info').get_json()
             except ResourceNotFound:
                 ddoc = json.loads(
                     binascii.a2b_base64(
@@ -478,10 +479,10 @@ class CouchDatabase(CommonBackend):
         This is achieved by creating a _security design document, see:
         http://docs.couchdb.org/en/latest/api/database/security.html
         """
-        security = self._database.security
+        security = self._database.resource.get_json('_security')[2]
         security['members'] = {'names': ['soledad'], 'roles': []}
         security['admins'] = {'names': [], 'roles': []}
-        self._database.security = security
+        self._database.resource.put_json('_security', body=security)
 
     def get_sync_target(self):
         """
@@ -888,7 +889,7 @@ class CouchDatabase(CommonBackend):
         try:
             resource = self._new_resource()
             resource.put_json(
-                doc.doc_id, body=buf.getvalue(), headers=envelope.headers)
+                doc.doc_id, body=str(buf.getvalue()), headers=envelope.headers)
         except ResourceConflict:
             raise RevisionConflict()
         if self.replica_uid + '_gen' in self.cache:
@@ -1337,6 +1338,17 @@ class CouchDatabase(CommonBackend):
                  in matching doc_ids order.
         :rtype: iterable
         """
+        # Workaround for:
+        #
+        #   http://bugs.python.org/issue7980
+        #   https://leap.se/code/issues/5449
+        #
+        # python-couchdb uses time.strptime, which is not thread safe. In
+        # order to avoid the problem described on the issues above, we preload
+        # strptime here by evaluating the conversion of an arbitrary date.
+        # This will not be needed when/if we switch from python-couchdb to
+        # paisley.
+        time.strptime('Mar 8 1917', '%b %d %Y')
         get_one = lambda doc_id: self._get_doc(doc_id, check_for_conflicts)
         docs = [THREAD_POOL.apply_async(get_one, [doc_id])
                 for doc_id in doc_ids]
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index d1a07a3a..b4797f5e 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -1507,9 +1507,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         will have the lowest privileged access to an user db.
         """
         self.create_db(ensure=False)
-        self.assertFalse(self.db._database.security)
+        self.assertFalse(self.db._database.resource.get_json('_security')[2])
         self.db.ensure_security_ddoc()
-        security_ddoc = self.db._database.security
+        security_ddoc = self.db._database.resource.get_json('_security')[2]
         self.assertIn('admins', security_ddoc)
         self.assertFalse(security_ddoc['admins']['names'])
         self.assertIn('members', security_ddoc)
diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py
index 7c6d281b..c48b511e 100644
--- a/scripts/db_access/reset_db.py
+++ b/scripts/db_access/reset_db.py
@@ -63,7 +63,7 @@ def get_url(empty):
     if empty is False:
         # get couch url
         cp = ConfigParser()
-        cp.read('/etc/leap/soledad-server.conf')
+        cp.read('/etc/soledad/soledad-server.conf')
         url = cp.get('soledad-server', 'couch_url')
     else:
         with open('/etc/couchdb/couchdb.netrc') as f:
diff --git a/scripts/db_access/server_side_db.py b/scripts/db_access/server_side_db.py
index 18641a0f..fcdd14b6 100644
--- a/scripts/db_access/server_side_db.py
+++ b/scripts/db_access/server_side_db.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 
 # This script gives server-side access to one Soledad user database by using
-# the configuration stored in /etc/leap/soledad-server.conf.
+# the configuration stored in /etc/soledad/soledad-server.conf.
 #
 # Use it like this:
 # 
@@ -20,7 +20,7 @@ uuid = sys.argv[1]
 
 # get couch url
 cp = ConfigParser()
-cp.read('/etc/leap/soledad-server.conf')
+cp.read('/etc/soledad/soledad-server.conf')
 url = cp.get('soledad-server', 'couch_url')
 
 # access user db
diff --git a/scripts/ddocs/update_design_docs.py b/scripts/ddocs/update_design_docs.py
index 2e2fa8f0..281482b8 100644
--- a/scripts/ddocs/update_design_docs.py
+++ b/scripts/ddocs/update_design_docs.py
@@ -50,7 +50,7 @@ def _parse_args():
 def _get_url():
     # get couch url
     cp = ConfigParser()
-    cp.read('/etc/leap/soledad-server.conf')
+    cp.read('/etc/soledad/soledad-server.conf')
     url = urlparse(cp.get('soledad-server', 'couch_url'))
     # get admin password
     netloc = re.sub('^.*@', '', url.netloc)
diff --git a/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py
index 02c68015..1a603fd0 100755
--- a/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py
+++ b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py
@@ -30,7 +30,7 @@ from socket import error as socket_error
 from leap.soledad.common.couch import CouchDatabase
 
 
-SOLEDAD_CONFIG_FILE = '/etc/leap/soledad-server.conf'
+SOLEDAD_CONFIG_FILE = '/etc/soledad/soledad-server.conf'
 PREFIX = '/tmp/soledad_test'
 LOG_FORMAT = '%(asctime)s %(levelname)s %(message)s'
 RETRIES = 3  # number of times to retry uploading a document of a certain
diff --git a/server/changes/change_soledad_configdir b/server/changes/change_soledad_configdir
new file mode 100644
index 00000000..710b9ac8
--- /dev/null
+++ b/server/changes/change_soledad_configdir
@@ -0,0 +1,2 @@
+o Moves config directory from /etc/leap to /etc/soledad
+  resolves #7509
diff --git a/server/changes/create_db_cmd b/server/changes/create_db_cmd
index cee0a935..964a7906 100644
--- a/server/changes/create_db_cmd
+++ b/server/changes/create_db_cmd
@@ -1,3 +1,4 @@
   o Adds a new config parameter 'create_cmd', which allows sysadmin to specify
   which command will create a database. That command was added in
   pkg/create-user-db and debian package automates steps needed for sudo access.
+  o Read netrc path from configuration file for create-user-db command. 
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index 1a7e77a7..7eafc945 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -21,6 +21,7 @@ import netrc
 import argparse
 from leap.soledad.common.couch import CouchDatabase
 from leap.soledad.common.couch import is_db_name_valid
+from leap.soledad.server import load_configuration
 
 
 description = """
@@ -30,7 +31,7 @@ This is meant to be used by Soledad Server.
 parser = argparse.ArgumentParser(description=description)
 parser.add_argument('dbname', metavar='user-d34db33f', type=str,
                     help='database name on the format user-{uuid4}')
-NETRC_PATH = '/etc/couchdb/couchdb-admin.netrc'
+NETRC_PATH = load_configuration('/etc/soledad/soledad-server.conf')['admin_netrc']
 
 
 def url_for_db(dbname):
diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip
index 3e1aa992..58834d0e 100644
--- a/server/pkg/requirements.pip
+++ b/server/pkg/requirements.pip
@@ -1,10 +1,11 @@
 configparser
-couchdb
 u1db
 routes
 PyOpenSSL
 twisted
-Beaker
+#pinned for wheezy compatibility
+Beaker==1.6.3 #wheezy
+couchdb==0.8 #wheezy
 
 # XXX -- fix me!
 # oauth is not strictly needed by us, but we need it until u1db adds it to its
diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server
index d127c5c5..da00e06b 100644
--- a/server/pkg/soledad-server
+++ b/server/pkg/soledad-server
@@ -11,12 +11,12 @@
 
 PATH=/sbin:/bin:/usr/sbin:/usr/bin
 PIDFILE=/var/run/soledad.pid
-RUNDIR=/var/lib/soledad/
 OBJ=leap.soledad.server.application
 LOGFILE=/var/log/soledad.log
 HTTPS_PORT=2424
-CERT_PATH=/etc/leap/soledad-server.pem
-PRIVKEY_PATH=/etc/leap/soledad-server.key
+CONFDIR=/etc/soledad
+CERT_PATH="${CONFDIR}/soledad-server.pem"
+PRIVKEY_PATH="${CONFDIR}/soledad-server.key"
 TWISTD_PATH=/usr/bin/twistd
 HOME=/var/lib/soledad/
 SSL_METHOD=SSLv23_METHOD
@@ -25,7 +25,7 @@ GROUP=soledad
 
 [ -r /etc/default/soledad ] && . /etc/default/soledad
 
-test -r /etc/leap/ || exit 0
+test -r ${CONFDIR} || exit 0
 
 . /lib/lsb/init-functions
 
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index bb1c6db0..f64d07bf 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -283,19 +283,20 @@ def load_configuration(file_path):
     @return: A dictionary with the configuration.
     @rtype: dict
     """
-    conf = {
+    defaults = {
         'couch_url': 'http://localhost:5984',
-        'create_cmd': None
+        'create_cmd': None,
+        'admin_netrc': '/etc/couchdb/couchdb-admin.netrc',
     }
     config = configparser.ConfigParser()
     config.read(file_path)
     if 'soledad-server' in config:
-        for key in conf:
+        for key in defaults:
             if key in config['soledad-server']:
-                conf[key] = config['soledad-server'][key]
+                defaults[key] = config['soledad-server'][key]
     # TODO: implement basic parsing/sanitization of options comming from
     # config file.
-    return conf
+    return defaults
 
 
 # ----------------------------------------------------------------------------
@@ -303,7 +304,7 @@ def load_configuration(file_path):
 # ----------------------------------------------------------------------------
 
 def application(environ, start_response):
-    conf = load_configuration('/etc/leap/soledad-server.conf')
+    conf = load_configuration('/etc/soledad/soledad-server.conf')
     state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'])
     # WSGI application that may be used by `twistd -web`
     application = GzipMiddleware(