Merge remote-tracking branch 'refs/remotes/drebs/feature/5386_improve-sync' into develop

14 files changed, 106 insertions, 79 deletions

diff --git a/common/changes/feature_5386_prevent-couch-backend-from-trying-to-create-db b/common/changes/feature_5386_prevent-couch-backend-from-trying-to-create-db
new file mode 100644
index 00000000..9bbac329
--- /dev/null
+++ b/common/changes/feature_5386_prevent-couch-backend-from-trying-to-create-db
@@ -0,0 +1 @@
+  o Prevent couch backend from always trying to create the database (#5386).
diff --git a/common/changes/feature_5386_prevent-uneeded-get-on-couch-server-state b/common/changes/feature_5386_prevent-uneeded-get-on-couch-server-state
new file mode 100644
index 00000000..95919873
--- /dev/null
+++ b/common/changes/feature_5386_prevent-uneeded-get-on-couch-server-state
@@ -0,0 +1,2 @@
+  o Prevent Couch Server State from making one uneeded GET request on
+    instantiation (#5386).
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index b836c997..8ed704ba 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -31,9 +31,10 @@ import threading
 
 from StringIO import StringIO
 from collections import defaultdict
+from urlparse import urljoin
 
 
-from couchdb.client import Server
+from couchdb.client import Server, Database
 from couchdb.http import (
     ResourceConflict,
     ResourceNotFound,
@@ -353,7 +354,7 @@ class CouchDatabase(CommonBackend):
                      release_fun):
             """
             :param db: The database from where to get the document.
-            :type db: u1db.Database
+            :type db: CouchDatabase
             :param doc_id: The doc_id of the document to be retrieved.
             :type doc_id: str
             :param check_for_conflicts: Whether the get_doc() method should
@@ -380,7 +381,7 @@ class CouchDatabase(CommonBackend):
             self._release_fun()
 
     @classmethod
-    def open_database(cls, url, create, ensure_ddocs=False):
+    def open_database(cls, url, create, replica_uid=None, ensure_ddocs=False):
         """
         Open a U1DB database using CouchDB as backend.
 
@@ -388,6 +389,8 @@ class CouchDatabase(CommonBackend):
         :type url: str
         :param create: should the replica be created if it does not exist?
         :type create: bool
+        :param replica_uid: an optional unique replica identifier
+        :type replica_uid: str
         :param ensure_ddocs: Ensure that the design docs exist on server.
         :type ensure_ddocs: bool
 
@@ -406,10 +409,11 @@ class CouchDatabase(CommonBackend):
         except ResourceNotFound:
             if not create:
                 raise DatabaseDoesNotExist()
-        return cls(url, dbname, ensure_ddocs=ensure_ddocs)
+            server.create(dbname)
+        return cls(url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs)
 
-    def __init__(self, url, dbname, replica_uid=None, full_commit=True,
-                 session=None, ensure_ddocs=True):
+    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True,
+                 session=None):
         """
         Create a new Couch data container.
 
@@ -419,31 +423,23 @@ class CouchDatabase(CommonBackend):
         :type dbname: str
         :param replica_uid: an optional unique replica identifier
         :type replica_uid: str
-        :param full_commit: turn on the X-Couch-Full-Commit header
-        :type full_commit: bool
-        :param session: an http.Session instance or None for a default session
-        :type session: http.Session
         :param ensure_ddocs: Ensure that the design docs exist on server.
         :type ensure_ddocs: bool
+        :param session: an http.Session instance or None for a default session
+        :type session: http.Session
         """
         # save params
         self._url = url
-        self._full_commit = full_commit
         if session is None:
             session = Session(timeout=COUCH_TIMEOUT)
         self._session = session
         self._factory = CouchDocument
         self._real_replica_uid = None
         # configure couch
-        self._server = Server(url=self._url,
-                              full_commit=self._full_commit,
-                              session=self._session)
         self._dbname = dbname
-        try:
-            self._database = self._server[self._dbname]
-        except ResourceNotFound:
-            self._server.create(self._dbname)
-            self._database = self._server[self._dbname]
+        self._database = Database(
+            urljoin(self._url, self._dbname),
+            self._session)
         if replica_uid is not None:
             self._set_replica_uid(replica_uid)
         if ensure_ddocs:
@@ -482,7 +478,8 @@ class CouchDatabase(CommonBackend):
         """
         Delete a U1DB CouchDB database.
         """
-        del(self._server[self._dbname])
+        server = Server(url=self._url)
+        del(server[self._dbname])
 
     def close(self):
         """
@@ -494,7 +491,6 @@ class CouchDatabase(CommonBackend):
         self._url = None
         self._full_commit = None
         self._session = None
-        self._server = None
         self._database = None
         return True
 
@@ -1489,9 +1485,9 @@ class CouchServerState(ServerState):
         :return: The CouchDatabase object.
         :rtype: CouchDatabase
         """
-        return CouchDatabase.open_database(
-            self._couch_url + '/' + dbname,
-            create=False,
+        return CouchDatabase(
+            self._couch_url,
+            dbname,
             ensure_ddocs=False)
 
     def ensure_database(self, dbname):
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 86bb4b93..77c46e61 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -25,6 +25,7 @@ import copy
 import shutil
 from base64 import b64decode
 from mock import Mock
+from urlparse import urljoin
 
 from couchdb.client import Server
 from u1db import errors as u1db_errors
@@ -151,8 +152,11 @@ class CouchDBTestCase(unittest.TestCase):
 class TestCouchBackendImpl(CouchDBTestCase):
 
     def test__allocate_doc_id(self):
-        db = couch.CouchDatabase('http://localhost:' + str(self.wrapper.port),
-                                 'u1db_tests', ensure_ddocs=True)
+        db = couch.CouchDatabase.open_database(
+            urljoin(
+                'http://localhost:' + str(self.wrapper.port), 'u1db_tests'),
+                create=True,
+                ensure_ddocs=True)
         doc_id1 = db._allocate_doc_id()
         self.assertTrue(doc_id1.startswith('D-'))
         self.assertEqual(34, len(doc_id1))
@@ -166,28 +170,35 @@ class TestCouchBackendImpl(CouchDBTestCase):
 
 def make_couch_database_for_test(test, replica_uid):
     port = str(test.wrapper.port)
-    return couch.CouchDatabase('http://localhost:' + port, replica_uid,
-                               replica_uid=replica_uid or 'test',
-                               ensure_ddocs=True)
+    return couch.CouchDatabase.open_database(
+        urljoin('http://localhost:' + port, replica_uid),
+        create=True,
+        replica_uid=replica_uid or 'test',
+        ensure_ddocs=True)
 
 
 def copy_couch_database_for_test(test, db):
     port = str(test.wrapper.port)
     couch_url = 'http://localhost:' + port
     new_dbname = db._replica_uid + '_copy'
-    new_db = couch.CouchDatabase(couch_url,
-                                 new_dbname,
-                                 replica_uid=db._replica_uid or 'test')
+    new_db = couch.CouchDatabase.open_database(
+        urljoin(couch_url, new_dbname),
+        create=True,
+        replica_uid=db._replica_uid or 'test')
     # copy all docs
     old_couch_db = Server(couch_url)[db._replica_uid]
     new_couch_db = Server(couch_url)[new_dbname]
     for doc_id in old_couch_db:
         doc = old_couch_db.get(doc_id)
+        # bypass u1db_config document
+        if doc_id == 'u1db_config':
+            pass
         # copy design docs
-        if ('u1db_rev' not in doc):
+        elif doc_id.startswith('_design'):
+            del doc['_rev']
             new_couch_db.save(doc)
         # copy u1db docs
-        else:
+        elif 'u1db_rev' in doc:
             new_doc = {
                 '_id': doc['_id'],
                 'u1db_transactions': doc['u1db_transactions'],
@@ -228,7 +239,7 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase):
     def setUp(self):
         test_backends.AllDatabaseTests.setUp(self)
         # save db info because of test_close
-        self._server = self.db._server
+        self._url = self.db._url
         self._dbname = self.db._dbname
 
     def tearDown(self):
@@ -238,7 +249,8 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase):
         if self.id() == \
                 'leap.soledad.common.tests.test_couch.CouchTests.' \
                 'test_close(couch)':
-            del(self._server[self._dbname])
+            server = Server(url=self._url)
+            del(server[self._dbname])
         else:
             self.db.delete_database()
         test_backends.AllDatabaseTests.tearDown(self)
@@ -355,10 +367,10 @@ from u1db.backends.inmemory import InMemoryIndex
 
 class IndexedCouchDatabase(couch.CouchDatabase):
 
-    def __init__(self, url, dbname, replica_uid=None, full_commit=True,
-                     session=None, ensure_ddocs=True):
-        old_class.__init__(self, url, dbname, replica_uid, full_commit,
-                           session, ensure_ddocs=ensure_ddocs)
+    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True,
+            session=None):
+        old_class.__init__(self, url, dbname, replica_uid=replica_uid, 
+                           ensure_ddocs=ensure_ddocs, session=session)
         self._indexes = {}
 
     def _put_doc(self, old_doc, doc):
@@ -467,8 +479,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
 
     def setUp(self):
         CouchDBTestCase.setUp(self)
-        self.db = couch.CouchDatabase(
-            'http://127.0.0.1:%d' % self.wrapper.port, 'test',
+        self.db = couch.CouchDatabase.open_database(
+            urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'),
+            create=True,
             ensure_ddocs=False)  # note that we don't enforce ddocs here
 
     def tearDown(self):
@@ -509,8 +522,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         Test that all methods that access design documents list functions
         will raise if the functions are not present.
         """
-        self.db = couch.CouchDatabase(
-            'http://127.0.0.1:%d' % self.wrapper.port, 'test',
+        self.db = couch.CouchDatabase.open_database(
+            urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'),
+            create=True,
             ensure_ddocs=True)
         # erase views from _design/transactions
         transactions = self.db._database['_design/transactions']
@@ -538,8 +552,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         Test that all methods that access design documents list functions
         will raise if the functions are not present.
         """
-        self.db = couch.CouchDatabase(
-            'http://127.0.0.1:%d' % self.wrapper.port, 'test',
+        self.db = couch.CouchDatabase.open_database(
+            urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'),
+            create=True,
             ensure_ddocs=True)
         # erase views from _design/transactions
         transactions = self.db._database['_design/transactions']
@@ -567,8 +582,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         Test that all methods that access design documents' named views  will
         raise if the views are not present.
         """
-        self.db = couch.CouchDatabase(
-            'http://127.0.0.1:%d' % self.wrapper.port, 'test',
+        self.db = couch.CouchDatabase.open_database(
+            urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'),
+            create=True,
             ensure_ddocs=True)
         # erase views from _design/docs
         docs = self.db._database['_design/docs']
@@ -608,8 +624,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         Test that all methods that access design documents will raise if the
         design docs are not present.
         """
-        self.db = couch.CouchDatabase(
-            'http://127.0.0.1:%d' % self.wrapper.port, 'test',
+        self.db = couch.CouchDatabase.open_database(
+            urljoin('http://127.0.0.1:%d' % self.wrapper.port, 'test'),
+            create=True,
             ensure_ddocs=True)
         # delete _design/docs
         del self.db._database['_design/docs']
diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
index 3c457cc5..3c219b91 100644
--- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
+++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
@@ -24,6 +24,10 @@ import mock
 import tempfile
 import threading
 
+
+from urlparse import urljoin
+
+
 from leap.soledad.client import Soledad
 from leap.soledad.common.couch import CouchDatabase, CouchServerState
 from leap.soledad.common.tests.test_couch import CouchDBTestCase
@@ -101,8 +105,11 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
         TestCaseWithServer.setUp(self)
         CouchDBTestCase.setUp(self)
         self._couch_url = 'http://localhost:' + str(self.wrapper.port)
-        self.db = CouchDatabase(
-            self._couch_url, 'user-user-uuid', replica_uid='replica')
+        self.db = CouchDatabase.open_database(
+            urljoin(self._couch_url, 'user-user-uuid'),
+            create=True,
+            replica_uid='replica',
+            ensure_ddocs=True)
         self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
 
     def tearDown(self):
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index f8d2a64f..6fe9211c 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -27,6 +27,7 @@ import mock
 import time
 import binascii
 
+from urlparse import urljoin
 
 from leap.common.testing.basetest import BaseLeapTest
 from leap.soledad.common.couch import (
@@ -56,7 +57,8 @@ from leap.soledad.server.auth import URLToAuthorization
 def _couch_ensure_database(self, dbname):
     db = CouchDatabase.open_database(
         self._couch_url + '/' + dbname,
-        create=True)
+        create=True,
+        ensure_ddocs=True)
     return db, db._replica_uid
 
 CouchServerState.ensure_database = _couch_ensure_database
@@ -352,11 +354,10 @@ class EncryptedSyncTestCase(
         self.assertEqual([], doclist)
         doc1 = sol1.create_doc(json.loads(simple_doc))
         # ensure remote db exists before syncing
-        db = CouchDatabase(
-            self._couch_url,
-            # the name of the user database is "user-<uuid>".
-            'user-user-uuid',
-        )
+        db = CouchDatabase.open_database(
+            urljoin(self._couch_url, 'user-user-uuid'),
+            create=True,
+            ensure_ddocs=True)
         # sync with server
         sol1._server_url = self.getURL()
         sol1.sync()
@@ -408,11 +409,10 @@ class EncryptedSyncTestCase(
         self.assertEqual([], doclist)
         doc1 = sol1.create_doc(json.loads(simple_doc))
         # ensure remote db exists before syncing
-        db = CouchDatabase(
-            self._couch_url,
-            # the name of the user database is "user-<uuid>".
-            'user-user-uuid',
-        )
+        db = CouchDatabase.open_database(
+            urljoin(self._couch_url, 'user-user-uuid'),
+            create=True,
+            ensure_ddocs=True)
         # sync with server
         sol1._server_url = self.getURL()
         sol1.sync()
@@ -468,11 +468,10 @@ class EncryptedSyncTestCase(
         content = binascii.hexlify(os.urandom(length/2))  # len() == length
         doc1 = sol1.create_doc({'data': content})
         # ensure remote db exists before syncing
-        db = CouchDatabase(
-            self._couch_url,
-            # the name of the user database is "user-<uuid>".
-            'user-user-uuid',
-        )
+        db = CouchDatabase.open_database(
+            urljoin(self._couch_url, 'user-user-uuid'),
+            create=True,
+            ensure_ddocs=True)
         # sync with server
         sol1._server_url = self.getURL()
         sol1.sync()
@@ -512,11 +511,10 @@ class EncryptedSyncTestCase(
         for i in range(0, number_of_docs):
             sol1.create_doc(json.loads(simple_doc))
         # ensure remote db exists before syncing
-        db = CouchDatabase(
-            self._couch_url,
-            # the name of the user database is "user-<uuid>".
-            'user-user-uuid',
-        )
+        db = CouchDatabase.open_database(
+            urljoin(self._couch_url, 'user-user-uuid'),
+            create=True,
+            ensure_ddocs=True)
         # sync with server
         sol1._server_url = self.getURL()
         sol1.sync()
@@ -558,8 +556,14 @@ class LockResourceTestCase(
         self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
         self._couch_url = 'http://localhost:' + str(self.wrapper.port)
         # create the databases
-        CouchDatabase(self._couch_url, 'shared')
-        CouchDatabase(self._couch_url, 'tokens')
+        CouchDatabase.open_database(
+            urljoin(self._couch_url, 'shared'),
+            create=True,
+            ensure_ddocs=True)
+        CouchDatabase.open_database(
+            urljoin(self._couch_url, 'tokens'),
+            create=True,
+            ensure_ddocs=True)
         self._state = CouchServerState(
             self._couch_url, 'shared', 'tokens')
 
@@ -567,10 +571,10 @@ class LockResourceTestCase(
         CouchDBTestCase.tearDown(self)
         TestCaseWithServer.tearDown(self)
         # delete remote database
-        db = CouchDatabase(
-            self._couch_url,
-            'shared',
-        )
+        db = CouchDatabase.open_database(
+            urljoin(self._couch_url, 'shared'),
+            create=True,
+            ensure_ddocs=True)
         db.delete_database()
 
     def test__try_obtain_filesystem_lock(self):
diff --git a/scripts/update_design_docs.py b/scripts/ddocs/update_design_docs.py
index e7b5a29c..e7b5a29c 100644
--- a/scripts/update_design_docs.py
+++ b/scripts/ddocs/update_design_docs.py
diff --git a/scripts/backends_cpu_usage/log_cpu_usage.py b/scripts/profiling/backends_cpu_usage/log_cpu_usage.py
index 2674e1ff..2674e1ff 100755
--- a/scripts/backends_cpu_usage/log_cpu_usage.py
+++ b/scripts/profiling/backends_cpu_usage/log_cpu_usage.py
diff --git a/scripts/backends_cpu_usage/movingaverage.py b/scripts/profiling/backends_cpu_usage/movingaverage.py
index bac1b3e1..bac1b3e1 100644
--- a/scripts/backends_cpu_usage/movingaverage.py
+++ b/scripts/profiling/backends_cpu_usage/movingaverage.py
diff --git a/scripts/backends_cpu_usage/plot.py b/scripts/profiling/backends_cpu_usage/plot.py
index 4e5083ad..4e5083ad 100755
--- a/scripts/backends_cpu_usage/plot.py
+++ b/scripts/profiling/backends_cpu_usage/plot.py
diff --git a/scripts/backends_cpu_usage/test_u1db_sync.py b/scripts/profiling/backends_cpu_usage/test_u1db_sync.py
index 26ef8f9f..26ef8f9f 100755
--- a/scripts/backends_cpu_usage/test_u1db_sync.py
+++ b/scripts/profiling/backends_cpu_usage/test_u1db_sync.py
diff --git a/scripts/doc_put_memory_usage/find_max_upload_size.py b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py
index 02c68015..02c68015 100755
--- a/scripts/doc_put_memory_usage/find_max_upload_size.py
+++ b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py
diff --git a/scripts/doc_put_memory_usage/get-mem.py b/scripts/profiling/doc_put_memory_usage/get-mem.py
index d64875fc..d64875fc 100755
--- a/scripts/doc_put_memory_usage/get-mem.py
+++ b/scripts/profiling/doc_put_memory_usage/get-mem.py
diff --git a/scripts/doc_put_memory_usage/plot-mem.py b/scripts/profiling/doc_put_memory_usage/plot-mem.py
index e24679a2..e24679a2 100755
--- a/scripts/doc_put_memory_usage/plot-mem.py
+++ b/scripts/profiling/doc_put_memory_usage/plot-mem.py