Merge remote-tracking branch 'refs/remotes/drebs/bug/5739_fix-multipart-put-problem-2' into develop

7 files changed, 40 insertions, 37 deletions

diff --git a/common/changes/bug_5739_fix-multipart-problem b/common/changes/bug_5739_fix-multipart-problem
new file mode 100644
index 00000000..449e09b8
--- /dev/null
+++ b/common/changes/bug_5739_fix-multipart-problem
@@ -0,0 +1,2 @@
+  o Use a dedicated HTTP resource for couch multipart PUTs to avoid bigcouch
+    bug (#5739).
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 3bc1f543..b51b32f3 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -40,7 +40,9 @@ from couchdb.http import (
     ResourceConflict,
     ResourceNotFound,
     ServerError,
-    Session as CouchHTTPSession,
+    Session,
+    urljoin as couch_urljoin,
+    Resource,
 )
 from u1db import query_parser, vectorclock
 from u1db.errors import (
@@ -333,17 +335,6 @@ class MultipartWriter(object):
                 self.headers[name] = value
 
 
-class Session(CouchHTTPSession):
-    """
-    An HTTP session that can be closed.
-    """
-
-    def close_connections(self):
-        for key, conns in list(self.conns.items()):
-            for conn in conns:
-                conn.close()
-
-
 @contextmanager
 def couch_server(url):
     """
@@ -359,7 +350,6 @@ def couch_server(url):
     session = Session(timeout=COUCH_TIMEOUT)
     server = Server(url=url, session=session)
     yield server
-    session.close_connections()
 
 
 class CouchDatabase(CommonBackend):
@@ -511,7 +501,6 @@ class CouchDatabase(CommonBackend):
         """
         with couch_server(self._url) as server:
             del(server[self._dbname])
-        self.close_connections()
 
     def close(self):
         """
@@ -520,20 +509,12 @@ class CouchDatabase(CommonBackend):
         :return: True if db was succesfully closed.
         :rtype: bool
         """
-        self.close_connections()
         self._url = None
         self._full_commit = None
         self._session = None
         self._database = None
         return True
 
-    def close_connections(self):
-        """
-        Close all open connections to the couch server.
-        """
-        if self._session is not None:
-            self._session.close_connections()
-
     def __del__(self):
         """
         Close the database upon garbage collection.
@@ -897,11 +878,9 @@ class CouchDatabase(CommonBackend):
         envelope.close()
         # try to save and fail if there's a revision conflict
         try:
-            self._database.resource.put_json(
+            resource = self._new_resource()
+            resource.put_json(
                 doc.doc_id, body=buf.getvalue(), headers=envelope.headers)
-            # What follows is a workaround for an ugly bug. See:
-            # https://leap.se/code/issues/5448
-            self.close_connections()
         except ResourceConflict:
             raise RevisionConflict()
 
@@ -1473,6 +1452,20 @@ class CouchDatabase(CommonBackend):
                 continue
             yield t._doc
 
+    def _new_resource(self, *path):
+        """
+        Return a new resource for accessing a couch database.
+
+        :return: A resource for accessing a couch database.
+        :rtype: couchdb.http.Resource
+        """
+        # Workaround for: https://leap.se/code/issues/5448
+        url = couch_urljoin(self._database.resource.url, *path)
+        resource = Resource(url, Session(timeout=COUCH_TIMEOUT))
+        resource.credentials = self._database.resource.credentials
+        resource.headers = self._database.resource.headers.copy()
+        return resource
+
 
 class CouchSyncTarget(CommonSyncTarget):
     """
diff --git a/common/src/leap/soledad/common/ddocs/syncs/updates/state.js b/common/src/leap/soledad/common/ddocs/syncs/updates/state.js
index cb2b6b7b..d62aeb40 100644
--- a/common/src/leap/soledad/common/ddocs/syncs/updates/state.js
+++ b/common/src/leap/soledad/common/ddocs/syncs/updates/state.js
@@ -29,6 +29,7 @@
  *     '_rev' '<str>',
  *     'ongoing_syncs': {
  *         '<source_replica_uid>': {
+ *             'sync_id': '<sync_id>',
  *             'seen_ids': [['<doc_id>', <at_gen>[, ...],
  *             'changes_to_return': {
  *                  'gen': <gen>,
@@ -59,17 +60,22 @@ function(doc, req) {
     // parse and validate incoming data
     var body = JSON.parse(req.body);
     if (body['source_replica_uid'] == null)
-        return [null, 'invalid data']
+        return [null, 'invalid data'];
     var source_replica_uid = body['source_replica_uid'];
 
+    if (body['sync_id'] == null)
+        return [null, 'invalid data'];
+    var sync_id = body['sync_id'];
+
     // trash outdated sync data for that replica if that exists
     if (doc['ongoing_syncs'][source_replica_uid] != null &&
-            doc['ongoing_syncs'][source_replica_uid] == null)
+            doc['ongoing_syncs'][source_replica_uid]['sync_id'] != sync_id)
         delete doc['ongoing_syncs'][source_replica_uid];
 
     // create an entry for that source replica
     if (doc['ongoing_syncs'][source_replica_uid] == null)
         doc['ongoing_syncs'][source_replica_uid] = {
+            'sync_id': sync_id,
             'seen_ids': {},
             'changes_to_return': null,
         };
diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js
index 04ceb2ec..94b7e767 100644
--- a/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js
+++ b/common/src/leap/soledad/common/ddocs/syncs/views/changes_to_return/map.js
@@ -2,14 +2,15 @@ function(doc) {
   if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null)
     for (var source_replica_uid in doc['ongoing_syncs']) {
       var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return'];
+      var sync_id = doc['ongoing_syncs'][source_replica_uid]['sync_id'];
       if (changes == null)
-        emit([source_replica_uid, 0], null);
+        emit([source_replica_uid, sync_id, 0], null);
       else if (changes.length == 0)
-        emit([source_replica_uid, 0], []);
+        emit([source_replica_uid, sync_id, 0], []);
       else
         for (var i = 0; i < changes['changes_to_return'].length; i++)
           emit(
-            [source_replica_uid, i],
+            [source_replica_uid, sync_id, i],
             {
               'gen': changes['gen'],
               'trans_id': changes['trans_id'],
diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js
index 34c65b3f..16118e88 100644
--- a/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js
+++ b/common/src/leap/soledad/common/ddocs/syncs/views/seen_ids/map.js
@@ -1,9 +1,11 @@
 function(doc) {
   if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null)
-    for (var source_replica_uid in doc['ongoing_syncs'])
+    for (var source_replica_uid in doc['ongoing_syncs']) {
+      var sync_id = doc['ongoing_syncs'][source_replica_uid]['sync_id'];
       emit(
-        source_replica_uid,
+        [source_replica_uid, sync_id],
         {
           'seen_ids': doc['ongoing_syncs'][source_replica_uid]['seen_ids'],
         });
+    }
 }
diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js
index 1d8f8e84..e88c6ebb 100644
--- a/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js
+++ b/common/src/leap/soledad/common/ddocs/syncs/views/state/map.js
@@ -2,11 +2,12 @@ function(doc) {
   if (doc['_id'] == 'u1db_sync_state' && doc['ongoing_syncs'] != null)
     for (var source_replica_uid in doc['ongoing_syncs']) {
       var changes = doc['ongoing_syncs'][source_replica_uid]['changes_to_return'];
+      var sync_id = doc['ongoing_syncs'][source_replica_uid]['sync_id'];
       if (changes == null)
-        emit(source_replica_uid, null);
+        emit([source_replica_uid, sync_id], null);
       else
         emit(
-          source_replica_uid,
+          [source_replica_uid, sync_id],
           {
             'gen': changes['gen'],
             'trans_id': changes['trans_id'],
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index a1fa9568..3b1e5a06 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -219,7 +219,6 @@ def copy_couch_database_for_test(test, db):
                     new_couch_db.put_attachment(new_doc, att,
                                                 filename=att_name)
     # cleanup connections to prevent file descriptor leaking
-    session.close_connections()
     return new_db
 
 
@@ -253,7 +252,6 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase):
             session = couch.Session()
             server = Server(url=self._url, session=session)
             del(server[self._dbname])
-            session.close_connections()
         else:
             self.db.delete_database()
         test_backends.AllDatabaseTests.tearDown(self)