Merge branch 'develop' into debian/platform-0.8

3 files changed, 29 insertions, 13 deletions

diff --git a/server/changes/feat_handle_send_batch_better b/server/changes/feat_handle_send_batch_better
new file mode 100644
index 00000000..6ee8688a
--- /dev/null
+++ b/server/changes/feat_handle_send_batch_better
@@ -0,0 +1 @@
+o Added two methods to start and finish a batch on backend. They can be used to change database behaviour, allowing batch operations to be optimized.
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 00e1e9fb..22894dac 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -104,6 +104,7 @@ from leap.soledad.server.sync import (
 )
 
 from leap.soledad.common import SHARED_DB_NAME
+from leap.soledad.common.backend import SoledadBackend
 from leap.soledad.common.couch.state import CouchServerState
 
 # ----------------------------------------------------------------------------
@@ -264,6 +265,7 @@ CONFIG_DEFAULTS = {
         'couch_url': 'http://localhost:5984',
         'create_cmd': None,
         'admin_netrc': '/etc/couchdb/couchdb-admin.netrc',
+        'batching': False
     },
     'database-security': {
         'members': ['soledad'],
@@ -285,18 +287,22 @@ def load_configuration(file_path):
     @rtype: dict
     """
     defaults = dict(CONFIG_DEFAULTS)
-    config = configparser.ConfigParser()
+    config = configparser.SafeConfigParser()
     config.read(file_path)
-    for section in defaults.keys():
-        if section in config:
-            for key in defaults[section]:
-                if key in config[section]:
-                    defaults[section][key] = config[section][key]
-    for key, value in defaults['database-security'].iteritems():
-        if type(value) is not unicode:
+    for section in defaults:
+        if not config.has_section(section):
             continue
-        defaults['database-security'][key] = \
-                [item.strip() for item in value.split(',')]
+        for key, value in defaults[section].items():
+            if not config.has_option(section, key):
+                continue
+            elif type(value) == bool:
+                defaults[section][key] = config.getboolean(section, key)
+            elif type(value) == list:
+                values = config.get(section, key).split(',')
+                values = [v.strip() for v in values]
+                defaults[section][key] = values
+            else:
+                defaults[section][key] = config.get(section, key)
     # TODO: implement basic parsing/sanitization of options comming from
     # config file.
     return defaults
@@ -310,6 +316,7 @@ def application(environ, start_response):
     conf = load_configuration('/etc/soledad/soledad-server.conf')
     conf = conf['soledad-server']
     state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'])
+    SoledadBackend.BATCH_SUPPORT = conf['batching']
     # WSGI application that may be used by `twistd -web`
     application = GzipMiddleware(
         SoledadTokenAuthMiddleware(SoledadApp(state)))
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index db25c406..96f65912 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -112,6 +112,14 @@ class SyncExchange(sync.SyncExchange):
             doc = self._db.get_doc(changed_doc_id, include_deleted=True)
             return_doc_cb(doc, gen, trans_id)
 
+    def batched_insert_from_source(self, entries, sync_id):
+        self._db.batch_start()
+        for entry in entries:
+            doc, gen, trans_id, number_of_docs, doc_idx = entry
+            self.insert_doc_from_source(doc, gen, trans_id, number_of_docs,
+                                        doc_idx, sync_id)
+        self._db.batch_end()
+
     def insert_doc_from_source(
             self, doc, source_gen, trans_id,
             number_of_docs=None, doc_idx=None, sync_id=None):
@@ -198,6 +206,7 @@ class SyncResource(http_app.SyncResource):
         self.sync_exch = self.sync_exchange_class(
             db, self.source_replica_uid, last_known_generation, sync_id)
         self._sync_id = sync_id
+        self._staging = []
 
     @http_app.http_method(content_as_args=True)
     def post_put(
@@ -225,9 +234,7 @@ class SyncResource(http_app.SyncResource):
         :type doc_idx: int
         """
         doc = Document(id, rev, content)
-        self.sync_exch.insert_doc_from_source(
-            doc, gen, trans_id, number_of_docs=number_of_docs,
-            doc_idx=doc_idx, sync_id=self._sync_id)
+        self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx))
 
     @http_app.http_method(received=int, content_as_args=True)
     def post_get(self, received):
@@ -266,6 +273,7 @@ class SyncResource(http_app.SyncResource):
         Return the current generation and transaction_id after inserting one
         incoming document.
         """
+        self.sync_exch.batched_insert_from_source(self._staging, self._sync_id)
         self.responder.content_type = 'application/x-soledad-sync-response'
         self.responder.start_response(200)
         self.responder.start_stream(),