From 0192f5923932ce738656c5b9ec25167a1b74386a Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sun, 30 Aug 2015 05:37:21 -0300
Subject: [pkg] add Beaker as a server dependency

Soledad server will use Beaker as cache provider, starting with
sync_state being in memory.
---
 server/pkg/requirements.pip | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip
index d75678b2..3e1aa992 100644
--- a/server/pkg/requirements.pip
+++ b/server/pkg/requirements.pip
@@ -4,6 +4,7 @@ u1db
 routes
 PyOpenSSL
 twisted
+Beaker
 
 # XXX -- fix me!
 # oauth is not strictly needed by us, but we need it until u1db adds it to its
-- 
cgit v1.2.3


From a10ae06bb33c6caaaf3a8474cc6d7b01c33abc9f Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sun, 30 Aug 2015 05:17:20 -0300
Subject: [feat] first draft of sync_state in memory

This commit changes sync_state to be in memory, with all tests passing.
The memory variable for now is a dict with each key composed by
source_replica_uid and sync_id, replicating CouchDB implementation. Next
steps includes migrating this to Beaker and refactor/clean up code.

Changed the module's INFO dict to use Beaker's caching and adapted
methods to get and save from it. Still needs refactoring, all tests
passes.

Beaker is now using memory as default; It is configurable, but we aren't
opening the possibility of config now for security. We need to check
what can be misconfigured first.

We are not sure if beaker will be the definitive solution for server
side caching. This change isolates it with more granularity.
In order to replace it, just change get_cache_for to return the proper
caching object using another implementation. This caching object is
supposed to behave as a dict.
---
 server/src/leap/soledad/server/caching.py |  32 ++++++
 server/src/leap/soledad/server/state.py   | 143 ++++++++++++++++++++++++
 server/src/leap/soledad/server/sync.py    | 173 +-----------------------------
 3 files changed, 177 insertions(+), 171 deletions(-)
 create mode 100644 server/src/leap/soledad/server/caching.py
 create mode 100644 server/src/leap/soledad/server/state.py

diff --git a/server/src/leap/soledad/server/caching.py b/server/src/leap/soledad/server/caching.py
new file mode 100644
index 00000000..cd5d8dd4
--- /dev/null
+++ b/server/src/leap/soledad/server/caching.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# caching.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Server side caching. Using beaker for now.
+"""
+from beaker.cache import CacheManager
+
+
+def setup_caching():
+    _cache_manager = CacheManager(type='memory')
+    return _cache_manager
+
+
+_cache_manager = setup_caching()
+
+
+def get_cache_for(key):
+    return _cache_manager.get_cache(key)
diff --git a/server/src/leap/soledad/server/state.py b/server/src/leap/soledad/server/state.py
new file mode 100644
index 00000000..d1225170
--- /dev/null
+++ b/server/src/leap/soledad/server/state.py
@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+# state.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Server side synchronization infrastructure.
+"""
+from leap.soledad.server import caching
+
+
+class ServerSyncState(object):
+    """
+    The state of one sync session, as stored on backend server.
+
+    On server side, the ongoing syncs metadata is maintained in a caching layer.
+    """
+
+    def __init__(self, source_replica_uid, sync_id):
+        """
+        Initialize the sync state object.
+
+        :param sync_id: The id of current sync
+        :type sync_id: str
+        :param source_replica_uid: The source replica uid
+        :type source_replica_uid: str
+        """
+        self._source_replica_uid = source_replica_uid
+        self._sync_id = sync_id
+        self._storage = caching.get_cache_for(source_replica_uid + sync_id)
+
+    def _put_dict_info(self, key, value):
+        """
+        Put some information about the sync state.
+
+        :param key: The key for the info to be put.
+        :type key: str
+        :param value: The value for the info to be put.
+        :type value: str
+        """
+        if key not in self._storage:
+            self._storage[key] = []
+        info_list = self._storage.get(key)
+        info_list.append(value)
+        self._storage[key] = info_list
+
+    def put_seen_id(self, seen_id, gen):
+        """
+        Put one seen id on the sync state.
+
+        :param seen_id: The doc_id of a document seen during sync.
+        :type seen_id: str
+        :param gen: The corresponding db generation for that document.
+        :type gen: int
+        """
+        self._put_dict_info(
+            'seen_id',
+            (seen_id, gen))
+
+    def seen_ids(self):
+        """
+        Return all document ids seen during the sync.
+
+        :return: A dict with doc ids seen during the sync.
+        :rtype: dict
+        """
+        if 'seen_id' in self._storage:
+            seen_ids = self._storage.get('seen_id')
+        else:
+            seen_ids = []
+        return dict(seen_ids)
+
+    def put_changes_to_return(self, gen, trans_id, changes_to_return):
+        """
+        Put the calculated changes to return in the backend sync state.
+
+        :param gen: The target database generation that will be synced.
+        :type gen: int
+        :param trans_id: The target database transaction id that will be
+                         synced.
+        :type trans_id: str
+        :param changes_to_return: A list of tuples with the changes to be
+                                  returned during the sync process.
+        :type changes_to_return: list
+        """
+        self._put_dict_info(
+            'changes_to_return',
+            {
+                'gen': gen,
+                'trans_id': trans_id,
+                'changes_to_return': changes_to_return,
+            }
+        )
+
+    def sync_info(self):
+        """
+        Return information about the current sync state.
+
+        :return: The generation and transaction id of the target database
+                 which will be synced, and the number of documents to return,
+                 or a tuple of Nones if those have not already been sent to
+                 server.
+        :rtype: tuple
+        """
+        info = self._storage.get('changes_to_return') if 'changes_to_return' in self._storage else None
+        gen = None
+        trans_id = None
+        number_of_changes = None
+        if info:
+            info = info[0]
+            gen = info['gen']
+            trans_id = info['trans_id']
+            number_of_changes = len(info['changes_to_return'])
+        return gen, trans_id, number_of_changes
+
+    def next_change_to_return(self, received):
+        """
+        Return the next change to be returned to the source syncing replica.
+
+        :param received: How many documents the source replica has already
+                         received during the current sync process.
+        :type received: int
+        """
+        info = self._storage.get('changes_to_return') if 'changes_to_return' in self._storage else None
+        gen = trans_id = next_change_to_return = None
+        if info:
+            info = info[0]
+            gen = info['gen']
+            trans_id = info['trans_id']
+            if received < len(info['changes_to_return']):
+                next_change_to_return = tuple(info['changes_to_return'][received])
+        return gen, trans_id, next_change_to_return
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index 18c4ee40..64f7e4e7 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -17,183 +17,15 @@
 """
 Server side synchronization infrastructure.
 """
-import json
-
-from leap.soledad.common.couch import CouchDatabase
 from u1db import sync, Document
 from u1db.remote import http_app
+from leap.soledad.server.state import ServerSyncState
 
 
 MAX_REQUEST_SIZE = 200  # in Mb
 MAX_ENTRY_SIZE = 200  # in Mb
 
 
-class ServerSyncState(object):
-    """
-    The state of one sync session, as stored on backend server.
-
-    This object performes queries to distinct design documents:
-
-        _design/syncs/_update/state
-        _design/syncs/_view/state
-        _design/syncs/_view/seen_ids
-        _design/syncs/_view/changes_to_return
-
-    On server side, the ongoing syncs metadata is maintained in a document
-    called 'u1db_sync_state'.
-    """
-
-    def __init__(self, db, source_replica_uid, sync_id):
-        """
-        Initialize the sync state object.
-
-        :param db: The target syncing database.
-        :type db: CouchDatabase.
-        :param source_replica_uid: CouchDatabase
-        :type source_replica_uid: str
-        """
-        self._db = db
-        self._source_replica_uid = source_replica_uid
-        self._sync_id = sync_id
-
-    def _key(self, key):
-        """
-        Format a key to be used on couch views.
-
-        :param key: The lookup key.
-        :type key: json serializable object
-
-        :return: The properly formatted key.
-        :rtype: str
-        """
-        return json.dumps(key, separators=(',', ':'))
-
-    def _put_info(self, key, value):
-        """
-        Put some information on the sync state document.
-
-        This method works in conjunction with the
-        _design/syncs/_update/state update handler couch backend.
-
-        :param key: The key for the info to be put.
-        :type key: str
-        :param value: The value for the info to be put.
-        :type value: str
-        """
-        ddoc_path = [
-            '_design', 'syncs', '_update', 'state',
-            'u1db_sync_state']
-        res = self._db._database.resource(*ddoc_path)
-        with CouchDatabase.sync_info_lock[self._db.replica_uid]:
-            res.put_json(
-                body={
-                    'sync_id': self._sync_id,
-                    'source_replica_uid': self._source_replica_uid,
-                    key: value,
-                },
-                headers={'content-type': 'application/json'})
-
-    def put_seen_id(self, seen_id, gen):
-        """
-        Put one seen id on the sync state document.
-
-        :param seen_id: The doc_id of a document seen during sync.
-        :type seen_id: str
-        :param gen: The corresponding db generation for that document.
-        :type gen: int
-        """
-        self._put_info(
-            'seen_id',
-            [seen_id, gen])
-
-    def seen_ids(self):
-        """
-        Return all document ids seen during the sync.
-
-        :return: A list with doc ids seen during the sync.
-        :rtype: list
-        """
-        ddoc_path = ['_design', 'syncs', '_view', 'seen_ids']
-        resource = self._db._database.resource(*ddoc_path)
-        response = resource.get_json(
-            key=self._key([self._source_replica_uid, self._sync_id]))
-        data = response[2]
-        if data['rows']:
-            entry = data['rows'].pop()
-            return entry['value']['seen_ids']
-        return []
-
-    def put_changes_to_return(self, gen, trans_id, changes_to_return):
-        """
-        Put the calculated changes to return in the backend sync state
-        document.
-
-        :param gen: The target database generation that will be synced.
-        :type gen: int
-        :param trans_id: The target database transaction id that will be
-                         synced.
-        :type trans_id: str
-        :param changes_to_return: A list of tuples with the changes to be
-                                  returned during the sync process.
-        :type changes_to_return: list
-        """
-        self._put_info(
-            'changes_to_return',
-            {
-                'gen': gen,
-                'trans_id': trans_id,
-                'changes_to_return': changes_to_return,
-            }
-        )
-
-    def sync_info(self):
-        """
-        Return information about the current sync state.
-
-        :return: The generation and transaction id of the target database
-                 which will be synced, and the number of documents to return,
-                 or a tuple of Nones if those have not already been sent to
-                 server.
-        :rtype: tuple
-        """
-        ddoc_path = ['_design', 'syncs', '_view', 'state']
-        resource = self._db._database.resource(*ddoc_path)
-        response = resource.get_json(
-            key=self._key([self._source_replica_uid, self._sync_id]))
-        data = response[2]
-        gen = None
-        trans_id = None
-        number_of_changes = None
-        if data['rows'] and data['rows'][0]['value'] is not None:
-            value = data['rows'][0]['value']
-            gen = value['gen']
-            trans_id = value['trans_id']
-            number_of_changes = value['number_of_changes']
-        return gen, trans_id, number_of_changes
-
-    def next_change_to_return(self, received):
-        """
-        Return the next change to be returned to the source syncing replica.
-
-        :param received: How many documents the source replica has already
-                         received during the current sync process.
-        :type received: int
-        """
-        ddoc_path = ['_design', 'syncs', '_view', 'changes_to_return']
-        resource = self._db._database.resource(*ddoc_path)
-        response = resource.get_json(
-            key=self._key(
-                [self._source_replica_uid, self._sync_id, received]))
-        data = response[2]
-        if not data['rows']:
-            return None, None, None
-        value = data['rows'][0]['value']
-        gen = value['gen']
-        trans_id = value['trans_id']
-        next_change_to_return = value['next_change_to_return']
-        return gen, trans_id, tuple(next_change_to_return)
-
-
 class SyncExchange(sync.SyncExchange):
 
     def __init__(self, db, source_replica_uid, last_known_generation, sync_id):
@@ -216,8 +48,7 @@ class SyncExchange(sync.SyncExchange):
         self.new_trans_id = None
         self._trace_hook = None
         # recover sync state
-        self._sync_state = ServerSyncState(
-            self._db, self.source_replica_uid, sync_id)
+        self._sync_state = ServerSyncState(self.source_replica_uid, sync_id)
 
     def find_changes_to_return(self, received):
         """
-- 
cgit v1.2.3


From 92b86079d34da5252b826d32afabafde84dab1af Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 4 Sep 2015 17:30:13 -0300
Subject: [refactor] couch.py should use superclass methods

_put_doc_if_newer is implemented on CommonBackend already. This was
copied over to CouchBackend just to add ensure conflicts. We can do this
before calling the super method instead.
---
 common/src/leap/soledad/common/couch.py | 91 +++++----------------------------
 1 file changed, 13 insertions(+), 78 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 1c762036..771db69e 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -103,6 +103,7 @@ class CouchDocument(SoledadDocument):
         SoledadDocument.__init__(self, doc_id, rev, json, has_conflicts)
         self.couch_rev = None
         self.transactions = None
+        self._conflicts = None
 
     def get_conflicts(self):
         """
@@ -111,7 +112,7 @@ class CouchDocument(SoledadDocument):
         :return: The conflicted versions of the document.
         :rtype: [CouchDocument]
         """
-        return self._conflicts
+        return self._conflicts or []
 
     def set_conflicts(self, conflicts):
         """
@@ -1203,10 +1204,10 @@ class CouchDatabase(CommonBackend):
         :param doc: The document to be put.
         :type doc: CouchDocument
         """
-        my_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
-        doc.prune_conflicts(
-            vectorclock.VectorClockRev(doc.rev), self._replica_uid)
-        doc.add_conflict(my_doc)
+        my_doc = self._get_doc(doc.doc_id)
+        self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev))
+        doc.add_conflict(self._factory(doc.doc_id, my_doc.rev, my_doc.get_json()))
+        doc.has_conflicts = True
         self._put_doc(my_doc, doc)
 
     def resolve_doc(self, doc, conflicted_doc_revs):
@@ -1320,30 +1321,14 @@ class CouchDatabase(CommonBackend):
         """
         if not isinstance(doc, CouchDocument):
             doc = self._factory(doc.doc_id, doc.rev, doc.get_json())
-        self._save_source_info(replica_uid, replica_gen,
-                               replica_trans_id, number_of_docs,
-                               doc_idx, sync_id)
         my_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
-        if my_doc is not None:
-            my_doc.set_conflicts(
-                self.get_doc_conflicts(my_doc.doc_id, my_doc.couch_rev))
-        state, save_doc = _process_incoming_doc(
-            my_doc, doc, save_conflict, self.replica_uid)
-        if save_doc:
-            self._put_doc(my_doc, save_doc)
-            doc.update(save_doc)
-        return state, self._get_generation()
-
-    def _save_source_info(self, replica_uid, replica_gen, replica_trans_id,
-                          number_of_docs, doc_idx, sync_id):
-        """
-        Validate and save source information.
-        """
-        self._validate_source(replica_uid, replica_gen, replica_trans_id)
-        self._set_replica_gen_and_trans_id(
-            replica_uid, replica_gen, replica_trans_id,
-            number_of_docs=number_of_docs, doc_idx=doc_idx,
-            sync_id=sync_id)
+        if my_doc:
+            doc.set_conflicts(my_doc.get_conflicts())
+        return CommonBackend._put_doc_if_newer(self, doc, save_conflict, replica_uid,
+                                               replica_gen, replica_trans_id)
+
+    def _put_and_update_indexes(self, cur_doc, doc):
+        self._put_doc(cur_doc, doc)
 
     def get_docs(self, doc_ids, check_for_conflicts=True,
                  include_deleted=False):
@@ -1494,53 +1479,3 @@ class CouchServerState(ServerState):
                              delete databases.
         """
         raise Unauthorized()
-
-
-def _process_incoming_doc(my_doc, other_doc, save_conflict, replica_uid):
-    """
-    Check document, save and return state.
-    """
-    # at this point, `doc` has arrived from the other syncing party, and
-    # we will decide what to do with it.
-    # First, we prepare the arriving doc to update couch database.
-    new_doc = CouchDocument(
-        other_doc.doc_id, other_doc.rev, other_doc.get_json())
-    if my_doc is None:
-        return 'inserted', new_doc
-    new_doc.couch_rev = my_doc.couch_rev
-    new_doc.set_conflicts(my_doc.get_conflicts())
-    # fetch conflicts because we will eventually manipulate them
-    # from now on, it works just like u1db sqlite backend
-    doc_vcr = vectorclock.VectorClockRev(new_doc.rev)
-    cur_vcr = vectorclock.VectorClockRev(my_doc.rev)
-    if doc_vcr.is_newer(cur_vcr):
-        rev = new_doc.rev
-        new_doc.prune_conflicts(doc_vcr, replica_uid)
-        if new_doc.rev != rev:
-            # conflicts have been autoresolved
-            return 'superseded', new_doc
-        else:
-            return'inserted', new_doc
-    elif new_doc.rev == my_doc.rev:
-        # magical convergence
-        return 'converged', None
-    elif cur_vcr.is_newer(doc_vcr):
-        # Don't add this to seen_ids, because we have something newer,
-        # so we should send it back, and we should not generate a
-        # conflict
-        other_doc.update(new_doc)
-        return 'superseded', None
-    elif my_doc.same_content_as(new_doc):
-        # the documents have been edited to the same thing at both ends
-        doc_vcr.maximize(cur_vcr)
-        doc_vcr.increment(replica_uid)
-        new_doc.rev = doc_vcr.as_str()
-        return 'superseded', new_doc
-    else:
-        if save_conflict:
-            new_doc.prune_conflicts(
-                vectorclock.VectorClockRev(new_doc.rev), replica_uid)
-            new_doc.add_conflict(my_doc)
-            return 'conflicted', new_doc
-        other_doc.update(new_doc)
-        return 'conflicted', None
-- 
cgit v1.2.3


From d39b408e92bb2fa6a2dac3835e2f5209d2eee94c Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 4 Sep 2015 19:37:32 -0300
Subject: [refactor] uses ThreadPool instead of own implementation

Python has a native ThreadPool implementation that fits our needs.
Changing it to use this instead and making some calls simpler.
---
 common/src/leap/soledad/common/couch.py | 99 ++++++---------------------------
 1 file changed, 18 insertions(+), 81 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 771db69e..421fbac1 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -26,13 +26,12 @@ import logging
 import binascii
 import time
 import sys
-import threading
 
 
 from StringIO import StringIO
-from collections import defaultdict
 from urlparse import urljoin
 from contextlib import contextmanager
+from multiprocessing.pool import ThreadPool
 
 
 from couchdb.client import Server, Database
@@ -362,57 +361,15 @@ def couch_server(url):
     yield server
 
 
+THREAD_POOL = ThreadPool(20)
+
+
 class CouchDatabase(CommonBackend):
 
     """
     A U1DB implementation that uses CouchDB as its persistence layer.
     """
 
-    # We spawn threads to parallelize the CouchDatabase.get_docs() method
-    MAX_GET_DOCS_THREADS = 20
-
-    update_handler_lock = defaultdict(threading.Lock)
-    sync_info_lock = defaultdict(threading.Lock)
-
-    class _GetDocThread(threading.Thread):
-
-        """
-        A thread that gets a document from a database.
-
-        TODO: switch this for a twisted deferred to thread. This depends on
-        replacing python-couchdb for paisley in this module.
-        """
-
-        def __init__(self, db, doc_id, check_for_conflicts,
-                     release_fun):
-            """
-            :param db: The database from where to get the document.
-            :type db: CouchDatabase
-            :param doc_id: The doc_id of the document to be retrieved.
-            :type doc_id: str
-            :param check_for_conflicts: Whether the get_doc() method should
-                                        check for existing conflicts.
-            :type check_for_conflicts: bool
-            :param release_fun: A function that releases a semaphore, to be
-                                called after the document is fetched.
-            :type release_fun: function
-            """
-            threading.Thread.__init__(self)
-            self._db = db
-            self._doc_id = doc_id
-            self._check_for_conflicts = check_for_conflicts
-            self._release_fun = release_fun
-            self._doc = None
-
-        def run(self):
-            """
-            Fetch the document, store it as a property, and call the release
-            function.
-            """
-            self._doc = self._db._get_doc(
-                self._doc_id, self._check_for_conflicts)
-            self._release_fun()
-
     @classmethod
     def open_database(cls, url, create, replica_uid=None, ensure_ddocs=False):
         """
@@ -477,9 +434,6 @@ class CouchDatabase(CommonBackend):
             self._set_replica_uid(replica_uid)
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
-        # initialize a thread pool for parallelizing get_docs()
-        self._sem_pool = threading.BoundedSemaphore(
-            value=self.MAX_GET_DOCS_THREADS)
 
     def ensure_ddocs_on_db(self):
         """
@@ -735,6 +689,9 @@ class CouchDatabase(CommonBackend):
                     attachments=True)[2]
         except ResourceNotFound:
             return None
+        return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts)
+
+    def __parse_doc_from_couch(self, result, doc_id, check_for_conflicts=False):
         # restrict to u1db documents
         if 'u1db_rev' not in result:
             return None
@@ -799,11 +756,8 @@ class CouchDatabase(CommonBackend):
         """
 
         generation = self._get_generation()
-        results = []
-        for row in self._database.view('_all_docs'):
-            doc = self.get_doc(row.id, include_deleted=include_deleted)
-            if doc is not None:
-                results.append(doc)
+        results = list(self.get_docs(self._database,
+                                     include_deleted=include_deleted))
         return (generation, results)
 
     def _put_doc(self, old_doc, doc):
@@ -1335,12 +1289,12 @@ class CouchDatabase(CommonBackend):
         """
         Get the JSON content for many documents.
 
-        :param doc_ids: A list of document identifiers.
+        :param doc_ids: A list of document identifiers or None for all.
         :type doc_ids: list
         :param check_for_conflicts: If set to False, then the conflict check
                                     will be skipped, and 'None' will be
                                     returned instead of True/False.
-        :type check_for_conflictsa: bool
+        :type check_for_conflicts: bool
         :param include_deleted: If set to True, deleted documents will be
                                 returned with empty content. Otherwise deleted
                                 documents will not be included in the results.
@@ -1348,31 +1302,14 @@ class CouchDatabase(CommonBackend):
                  in matching doc_ids order.
         :rtype: iterable
         """
-        # Workaround for:
-        #
-        #   http://bugs.python.org/issue7980
-        #   https://leap.se/code/issues/5449
-        #
-        # python-couchdb uses time.strptime, which is not thread safe. In
-        # order to avoid the problem described on the issues above, we preload
-        # strptime here by evaluating the conversion of an arbitrary date.
-        # This will not be needed when/if we switch from python-couchdb to
-        # paisley.
-        time.strptime('Mar 8 1917', '%b %d %Y')
-        # spawn threads to retrieve docs
-        threads = []
-        for doc_id in doc_ids:
-            self._sem_pool.acquire()
-            t = self._GetDocThread(self, doc_id, check_for_conflicts,
-                                   self._sem_pool.release)
-            t.start()
-            threads.append(t)
-        # join threads and yield docs
-        for t in threads:
-            t.join()
-            if t._doc.is_tombstone() and not include_deleted:
+        get_one = lambda doc_id: self._get_doc(doc_id, check_for_conflicts)
+        docs = [THREAD_POOL.apply_async(get_one, [doc_id])
+                for doc_id in doc_ids]
+        for doc in docs:
+            doc = doc.get()
+            if not doc or not include_deleted and doc.is_tombstone():
                 continue
-            yield t._doc
+            yield doc
 
     def _prune_conflicts(self, doc, doc_vcr):
         """
-- 
cgit v1.2.3


From f9faa007bc0d5d681f85aa246ea05ec5fe35ccc5 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sun, 6 Sep 2015 05:39:05 -0300
Subject: [feat] adds caching for other gen and trans id

There are two functions in couch.py used to save and retrieve the last
know gen and trans id for the syncing replica. The get function is
called very often, but is only set on one point. Added a simple caching
to avoid queying couch for a value that we already have.
If cache is empty, it just query as usual and fills it.
---
 common/src/leap/soledad/common/couch.py    | 11 +++++++++--
 server/src/leap/soledad/server/__init__.py |  4 +++-
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 421fbac1..3b120cd1 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -434,6 +434,7 @@ class CouchDatabase(CommonBackend):
             self._set_replica_uid(replica_uid)
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
+        self.cache = {}
 
     def ensure_ddocs_on_db(self):
         """
@@ -1047,6 +1048,8 @@ class CouchDatabase(CommonBackend):
                  synchronized with the replica, this is (0, '').
         :rtype: (int, str)
         """
+        if other_replica_uid in self.cache:
+            return self.cache[other_replica_uid]
         # query a couch view
         result = self._database.view('syncs/log')
         if len(result[other_replica_uid].rows) == 0:
@@ -1129,6 +1132,7 @@ class CouchDatabase(CommonBackend):
                                              design document for an yet
                                              unknown reason.
         """
+        self.cache[other_replica_uid] = (other_generation, other_transaction_id)
         # query a couch update function
         ddoc_path = ['_design', 'syncs', '_update', 'put', 'u1db_sync_log']
         res = self._database.resource(*ddoc_path)
@@ -1362,7 +1366,7 @@ class CouchServerState(ServerState):
     Inteface of the WSGI server with the CouchDB backend.
     """
 
-    def __init__(self, couch_url):
+    def __init__(self, couch_url, cache=None):
         """
         Initialize the couch server state.
 
@@ -1370,6 +1374,7 @@ class CouchServerState(ServerState):
         :type couch_url: str
         """
         self.couch_url = couch_url
+        self.cache = cache or {}
 
     def open_database(self, dbname):
         """
@@ -1381,10 +1386,12 @@ class CouchServerState(ServerState):
         :return: The CouchDatabase object.
         :rtype: CouchDatabase
         """
-        return CouchDatabase(
+        db = CouchDatabase(
             self.couch_url,
             dbname,
             ensure_ddocs=False)
+        db.cache = self.cache
+        return db
 
     def ensure_database(self, dbname):
         """
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 1b795016..58f2b0b1 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -111,6 +111,7 @@ from leap.soledad.server.sync import (
 
 from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common.couch import CouchServerState
+from leap.soledad.server.caching import get_cache_for
 
 old_tsafe = tsafe
 
@@ -303,7 +304,8 @@ def load_configuration(file_path):
 
 def application(environ, start_response):
     conf = load_configuration('/etc/leap/soledad-server.conf')
-    state = CouchServerState(conf['couch_url'])
+    cache = get_cache_for('replica_cache')
+    state = CouchServerState(conf['couch_url'], cache=cache)
     # WSGI application that may be used by `twistd -web`
     application = GzipMiddleware(
         SoledadTokenAuthMiddleware(SoledadApp(state)))
-- 
cgit v1.2.3


From 14300959a12e4908df7b00281d2590e627a47ba9 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 7 Sep 2015 03:56:02 -0300
Subject: [feat] full caching for each sync_id session

The CouchDB backend implementation was accessing CouchDB too many times
for the same values. Those values are known inside the same sync_id,
which is the id of current sync session.
This commit adds caching for all redundant calls to Couch inside the
same sync_id for each replica.
Refactoring is still needed, but for now couch.py works normally as if
caching is not present, while sync.py injects the cache as a attribute
to enable it. This needs a simpler implementation.
---
 common/src/leap/soledad/common/couch.py    | 27 +++++++++++++++++++++++----
 server/src/leap/soledad/server/__init__.py |  4 +---
 server/src/leap/soledad/server/sync.py     |  3 +++
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 3b120cd1..8c152353 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -434,7 +434,14 @@ class CouchDatabase(CommonBackend):
             self._set_replica_uid(replica_uid)
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
-        self.cache = {}
+        self._cache = None
+
+    @property
+    def cache(self):
+        if self._cache is not None:
+            return self._cache
+        else:
+            return {}
 
     def ensure_ddocs_on_db(self):
         """
@@ -513,7 +520,10 @@ class CouchDatabase(CommonBackend):
         :rtype: str
         """
         if self._real_replica_uid is not None:
+            self.cache[self._url] = {'replica_uid': self._real_replica_uid}
             return self._real_replica_uid
+        if self._url in self.cache:
+            return self.cache[self._url]['replica_uid']
         try:
             # grab replica_uid from server
             doc = self._database['u1db_config']
@@ -551,10 +561,13 @@ class CouchDatabase(CommonBackend):
                                              unknown reason.
         """
         # query a couch list function
+        if self.replica_uid + '_gen' in self.cache:
+            return self.cache[self.replica_uid + '_gen']['generation']
         ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
         res = self._database.resource(*ddoc_path)
         try:
             response = res.get_json()
+            self.cache[self.replica_uid + '_gen'] = response[2]
             return response[2]['generation']
         except ResourceNotFound as e:
             raise_missing_design_doc_error(e, ddoc_path)
@@ -582,11 +595,15 @@ class CouchDatabase(CommonBackend):
                                              design document for an yet
                                              unknown reason.
         """
+        if self.replica_uid + '_gen' in self.cache:
+            response = self.cache[self.replica_uid + '_gen']
+            return (response['generation'], response['transaction_id'])
         # query a couch list function
         ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
         res = self._database.resource(*ddoc_path)
         try:
             response = res.get_json()
+            self.cache[self.replica_uid + '_gen'] = response[2]
             return (response[2]['generation'], response[2]['transaction_id'])
         except ResourceNotFound as e:
             raise_missing_design_doc_error(e, ddoc_path)
@@ -846,6 +863,10 @@ class CouchDatabase(CommonBackend):
                 doc.doc_id, body=buf.getvalue(), headers=envelope.headers)
         except ResourceConflict:
             raise RevisionConflict()
+        if self.replica_uid + '_gen' in self.cache:
+            gen_info = self.cache[self.replica_uid + '_gen']
+            gen_info['generation'] += 1
+            gen_info['transaction_id'] = transactions[-1][1]
 
     def put_doc(self, doc):
         """
@@ -1366,7 +1387,7 @@ class CouchServerState(ServerState):
     Inteface of the WSGI server with the CouchDB backend.
     """
 
-    def __init__(self, couch_url, cache=None):
+    def __init__(self, couch_url):
         """
         Initialize the couch server state.
 
@@ -1374,7 +1395,6 @@ class CouchServerState(ServerState):
         :type couch_url: str
         """
         self.couch_url = couch_url
-        self.cache = cache or {}
 
     def open_database(self, dbname):
         """
@@ -1390,7 +1410,6 @@ class CouchServerState(ServerState):
             self.couch_url,
             dbname,
             ensure_ddocs=False)
-        db.cache = self.cache
         return db
 
     def ensure_database(self, dbname):
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 58f2b0b1..1b795016 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -111,7 +111,6 @@ from leap.soledad.server.sync import (
 
 from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common.couch import CouchServerState
-from leap.soledad.server.caching import get_cache_for
 
 old_tsafe = tsafe
 
@@ -304,8 +303,7 @@ def load_configuration(file_path):
 
 def application(environ, start_response):
     conf = load_configuration('/etc/leap/soledad-server.conf')
-    cache = get_cache_for('replica_cache')
-    state = CouchServerState(conf['couch_url'], cache=cache)
+    state = CouchServerState(conf['couch_url'])
     # WSGI application that may be used by `twistd -web`
     application = GzipMiddleware(
         SoledadTokenAuthMiddleware(SoledadApp(state)))
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index 64f7e4e7..262b6769 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -20,6 +20,7 @@ Server side synchronization infrastructure.
 from u1db import sync, Document
 from u1db.remote import http_app
 from leap.soledad.server.state import ServerSyncState
+from leap.soledad.server.caching import get_cache_for
 
 
 MAX_REQUEST_SIZE = 200  # in Mb
@@ -188,6 +189,8 @@ class SyncResource(http_app.SyncResource):
             db, self.replica_uid = self.state.ensure_database(self.dbname)
         else:
             db = self.state.open_database(self.dbname)
+        cache = get_cache_for('db-' + sync_id + db.replica_uid)
+        db._cache = cache
         # validate the information the client has about server replica
         db.validate_gen_and_trans_id(
             last_known_generation, last_known_trans_id)
-- 
cgit v1.2.3


From fc5f4ae965ca48946af9f6982b2719562168131c Mon Sep 17 00:00:00 2001
From: drebs <drebs@leap.se>
Date: Fri, 28 Aug 2015 17:25:41 -0300
Subject: [feat] use one doc per remote replica in sync log

Before this change, we used a complicated update handler for storing the sync
state on the couchdb backend. That update handler was implemented as an
attempt to make couchdb take care of some validation for the update of the
sync log during the sync exchange, mainly to allow concurrent received
documents insertion during a sync.

Right now we rely on the remote sending one document at a time and do not
support concurrent insertions in the remote database backed by couch. Because
of that, the code removed by this commit was unneeded. And more: it was a
bottleneck of the sync process because we were writing to an unique file and
using unnecessary couch design docs processing for that. So this commit both
simplifies the storage of remote sync and removes a bottleneck of the sync
process.

Conflicts:
	common/src/leap/soledad/common/couch.py
	common/src/leap/soledad/common/tests/test_couch.py
---
 common/src/leap/soledad/common/couch.py            |  63 +++------
 .../leap/soledad/common/ddocs/syncs/updates/put.js | 151 ---------------------
 .../soledad/common/ddocs/syncs/views/log/map.js    |  12 --
 common/src/leap/soledad/common/tests/test_couch.py |   8 --
 .../tests/test_couch_operations_atomicity.py       |  53 +++-----
 5 files changed, 42 insertions(+), 245 deletions(-)
 delete mode 100644 common/src/leap/soledad/common/ddocs/syncs/updates/put.js
 delete mode 100644 common/src/leap/soledad/common/ddocs/syncs/views/log/map.js

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 8c152353..83f542ab 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -1071,14 +1071,20 @@ class CouchDatabase(CommonBackend):
         """
         if other_replica_uid in self.cache:
             return self.cache[other_replica_uid]
-        # query a couch view
-        result = self._database.view('syncs/log')
-        if len(result[other_replica_uid].rows) == 0:
-            return (0, '')
-        return (
-            result[other_replica_uid].rows[0]['value']['known_generation'],
-            result[other_replica_uid].rows[0]['value']['known_transaction_id']
-        )
+
+        doc_id = 'u1db_sync_%s' % other_replica_uid
+        try:
+            doc = self._database[doc_id]
+        except ResourceNotFound:
+            doc = {
+                '_id': doc_id,
+                'generation': 0,
+                'transaction_id': '',
+            }
+            self._database.save(doc)
+        result = doc['generation'], doc['transaction_id']
+        self.cache[other_replica_uid] = result
+        return result
 
     def _set_replica_gen_and_trans_id(self, other_replica_uid,
                                       other_generation, other_transaction_id,
@@ -1138,43 +1144,16 @@ class CouchDatabase(CommonBackend):
         :type doc_idx: int
         :param sync_id: The id of the current sync session.
         :type sync_id: str
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
         """
         self.cache[other_replica_uid] = (other_generation, other_transaction_id)
-        # query a couch update function
-        ddoc_path = ['_design', 'syncs', '_update', 'put', 'u1db_sync_log']
-        res = self._database.resource(*ddoc_path)
+        doc_id = 'u1db_sync_%s' % other_replica_uid
         try:
-            with CouchDatabase.update_handler_lock[self._get_replica_uid()]:
-                body = {
-                    'other_replica_uid': other_replica_uid,
-                    'other_generation': other_generation,
-                    'other_transaction_id': other_transaction_id,
-                }
-                if number_of_docs is not None:
-                    body['number_of_docs'] = number_of_docs
-                if doc_idx is not None:
-                    body['doc_idx'] = doc_idx
-                if sync_id is not None:
-                    body['sync_id'] = sync_id
-                res.put_json(
-                    body=body,
-                    headers={'content-type': 'application/json'})
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
+            doc = self._database[doc_id]
+        except ResourceNotFound:
+            doc = {'_id': doc_id}
+        doc['generation'] = other_generation
+        doc['transaction_id'] = other_transaction_id
+        self._database.save(doc)
 
     def _force_doc_sync_conflict(self, doc):
         """
diff --git a/common/src/leap/soledad/common/ddocs/syncs/updates/put.js b/common/src/leap/soledad/common/ddocs/syncs/updates/put.js
deleted file mode 100644
index b0ae2de6..00000000
--- a/common/src/leap/soledad/common/ddocs/syncs/updates/put.js
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * The u1db_sync_log document stores both the actual sync log and a list of
- * pending updates to the log, in case we receive incoming documents out of
- * the correct order (i.e. if there are parallel PUTs during the sync
- * process).
- *
- * The structure of the document is the following:
- *
- *     {
- *         'syncs': [
- *             ['<replica_uid>', <gen>, '<trans_id>'],
- *             ... 
- *         ],
- *         'pending': {
- *             'other_replica_uid': {
- *                 'sync_id': '<sync_id>',
- *                 'log': [[<gen>, '<trans_id>'], ...]
- *             },
- *             ...
- *         }
- *     }
- *
- * The update function below does the following:
- *
- *   0. If we do not receive a sync_id, we just update the 'syncs' list with
- *      the incoming info about the source replica state.
- *
- *   1. Otherwise, if the incoming sync_id differs from current stored
- *      sync_id, then we assume that the previous sync session for that source
- *      replica was interrupted and discard all pending data.
- *
- *   2. Then we append incoming info as pending data for that source replica
- *      and current sync_id, and sort the pending data by generation.
- *
- *   3. Then we go through pending data and find the most recent generation
- *      that we can use to update the actual sync log.
- *
- *   4. Finally, we insert the most up to date information into the sync log.
- */
-function(doc, req){
-
-    // create the document if it doesn't exist
-    if (!doc) {
-        doc = {}
-        doc['_id'] = 'u1db_sync_log';
-        doc['syncs'] = [];
-    }
-
-    // get and validate incoming info
-    var body = JSON.parse(req.body);
-    var other_replica_uid = body['other_replica_uid'];
-    var other_generation = parseInt(body['other_generation']);
-    var other_transaction_id = body['other_transaction_id']
-    var sync_id = body['sync_id'];
-    var number_of_docs = body['number_of_docs'];
-    var doc_idx = body['doc_idx'];
-
-    // parse integers
-    if (number_of_docs != null)
-        number_of_docs = parseInt(number_of_docs);
-    if (doc_idx != null)
-        doc_idx = parseInt(doc_idx);
-
-    if (other_replica_uid == null
-            || other_generation == null
-            || other_transaction_id == null)
-        return [null, 'invalid data'];
-
-    // create slot for pending logs
-    if (doc['pending'] == null)
-        doc['pending'] = {};
-
-    // these are the values that will be actually inserted
-    var current_gen = other_generation;
-    var current_trans_id = other_transaction_id;
-
-    /*------------- Wait for sequential values before storing -------------*/
-
-    // we just try to obtain pending log if we received a sync_id
-    if (sync_id != null) {
-
-        // create slot for current source and sync_id pending log
-        if (doc['pending'][other_replica_uid] == null
-                || doc['pending'][other_replica_uid]['sync_id'] != sync_id) {
-            doc['pending'][other_replica_uid] = {
-                'sync_id': sync_id,
-                'log': [],
-                'last_doc_idx': 0,
-            }
-        }
-
-        // append incoming data to pending log
-        doc['pending'][other_replica_uid]['log'].push([
-            other_generation,
-            other_transaction_id,
-            doc_idx,
-        ])
-
-        // sort pending log according to generation
-        doc['pending'][other_replica_uid]['log'].sort(function(a, b) {
-            return a[0] - b[0];
-        });
-
-        // get most up-to-date information from pending log
-        var last_doc_idx = doc['pending'][other_replica_uid]['last_doc_idx'];
-        var pending_idx = doc['pending'][other_replica_uid]['log'][0][2];
-
-        current_gen = null;
-        current_trans_id = null;
-
-        while (last_doc_idx + 1 == pending_idx) {
-            pending = doc['pending'][other_replica_uid]['log'].shift()
-            current_gen = pending[0];
-            current_trans_id = pending[1];
-            last_doc_idx = pending[2]
-            if (doc['pending'][other_replica_uid]['log'].length == 0)
-                break;
-            pending_idx = doc['pending'][other_replica_uid]['log'][0][2];
-        }
-
-        // leave the sync log untouched if we still did not receive enough docs
-        if (current_gen == null)
-            return [doc, 'ok'];
-
-        // update last index of received doc
-        doc['pending'][other_replica_uid]['last_doc_idx'] = last_doc_idx;
-
-        // eventually remove all pending data from that replica
-        if (last_doc_idx == number_of_docs)
-            delete doc['pending'][other_replica_uid]
-    }
-
-    /*--------------- Store source replica info on sync log ---------------*/
-
-    // remove outdated info
-    doc['syncs'] = doc['syncs'].filter(
-        function (entry) {
-            return entry[0] != other_replica_uid;
-        }
-    );
-
-    // store in log
-    doc['syncs'].push([
-        other_replica_uid,
-        current_gen,
-        current_trans_id 
-    ]);
-
-    return [doc, 'ok'];
-}
-
diff --git a/common/src/leap/soledad/common/ddocs/syncs/views/log/map.js b/common/src/leap/soledad/common/ddocs/syncs/views/log/map.js
deleted file mode 100644
index a63c7cf4..00000000
--- a/common/src/leap/soledad/common/ddocs/syncs/views/log/map.js
+++ /dev/null
@@ -1,12 +0,0 @@
-function(doc) {
-    if (doc._id == 'u1db_sync_log') {
-        if (doc.syncs)
-            doc.syncs.forEach(function (entry) {
-                emit(entry[0],
-                    {
-                        'known_generation': entry[1],
-                        'known_transaction_id': entry[2]
-                    });
-            });
-    }
-}
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index a08ffd16..c8d13667 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -1359,10 +1359,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         self.assertRaises(
             errors.MissingDesignDocError,
             self.db.whats_changed)
-        # _do_set_replica_gen_and_trans_id()
-        self.assertRaises(
-            errors.MissingDesignDocError,
-            self.db._do_set_replica_gen_and_trans_id, 1, 2, 3)
 
     def test_missing_design_doc_functions_raises(self):
         """
@@ -1489,10 +1485,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         self.assertRaises(
             errors.MissingDesignDocDeletedError,
             self.db.whats_changed)
-        # _do_set_replica_gen_and_trans_id()
-        self.assertRaises(
-            errors.MissingDesignDocDeletedError,
-            self.db._do_set_replica_gen_and_trans_id, 1, 2, 3)
 
     def test_ensure_ddoc_independently(self):
         """
diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
index 25f709ca..3e8e8cce 100644
--- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
+++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
@@ -163,6 +163,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
                 2,
                 len(filter(lambda t: t[0] == doc_id, transaction_log)))
 
+    @defer.inlineCallbacks
     def test_correct_sync_log_after_sequential_syncs(self):
         """
         Assert that the sync_log increases accordingly with sequential syncs.
@@ -170,21 +171,21 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
         sol = self._soledad_instance(
             auth_token='auth-token',
             server_url=self.getURL())
+        source_replica_uid = sol._dbpool.replica_uid
 
-        def _create_docs(results):
+        def _create_docs():
             deferreds = []
             for i in xrange(0, REPEAT_TIMES):
                 deferreds.append(sol.create_doc({}))
-            return defer.DeferredList(deferreds)
+            return defer.gatherResults(deferreds)
 
         def _assert_transaction_and_sync_logs(results, sync_idx):
             # assert sizes of transaction and sync logs
             self.assertEqual(
                 sync_idx * REPEAT_TIMES,
                 len(self.db._get_transaction_log()))
-            self.assertEqual(
-                1 if sync_idx > 0 else 0,
-                len(self.db._database.view('syncs/log').rows))
+            gen, _ = self.db._get_replica_gen_and_trans_id(source_replica_uid)
+            self.assertEqual(sync_idx * REPEAT_TIMES, gen)
 
         def _assert_sync(results, sync_idx):
             gen, docs = results
@@ -193,40 +194,28 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
             # assert sizes of transaction and sync logs
             self.assertEqual((sync_idx + 1) * REPEAT_TIMES,
                              len(self.db._get_transaction_log()))
-            sync_log_rows = self.db._database.view('syncs/log').rows
-            sync_log = sync_log_rows[0].value
-            replica_uid = sync_log_rows[0].key
-            known_gen = sync_log['known_generation']
-            known_trans_id = sync_log['known_transaction_id']
-            # assert sync_log has exactly 1 row
-            self.assertEqual(1, len(sync_log_rows))
-            # assert it has the correct replica_uid, gen and trans_id
-            self.assertEqual(sol._dbpool.replica_uid, replica_uid)
+            target_known_gen, target_known_trans_id = \
+                self.db._get_replica_gen_and_trans_id(source_replica_uid)
+            # assert it has the correct gen and trans_id
             conn_key = sol._dbpool._u1dbconnections.keys().pop()
             conn = sol._dbpool._u1dbconnections[conn_key]
             sol_gen, sol_trans_id = conn._get_generation_info()
-            self.assertEqual(sol_gen, known_gen)
-            self.assertEqual(sol_trans_id, known_trans_id)
-
-        # create some documents
-        d = _create_docs(None)
+            self.assertEqual(sol_gen, target_known_gen)
+            self.assertEqual(sol_trans_id, target_known_trans_id)
 
         # sync first time and assert success
-        d.addCallback(_assert_transaction_and_sync_logs, 0)
-        d.addCallback(lambda _: sol.sync())
-        d.addCallback(lambda _: sol.get_all_docs())
-        d.addCallback(_assert_sync, 0)
+        results = yield _create_docs()
+        _assert_transaction_and_sync_logs(results, 0)
+        yield sol.sync()
+        results = yield sol.get_all_docs()
+        _assert_sync(results, 0)
 
         # create more docs, sync second time and assert success
-        d.addCallback(_create_docs)
-        d.addCallback(_assert_transaction_and_sync_logs, 1)
-        d.addCallback(lambda _: sol.sync())
-        d.addCallback(lambda _: sol.get_all_docs())
-        d.addCallback(_assert_sync, 1)
-
-        d.addCallback(lambda _: sol.close())
-
-        return d
+        results = yield _create_docs()
+        _assert_transaction_and_sync_logs(results, 1)
+        yield sol.sync()
+        results = yield sol.get_all_docs()
+        _assert_sync(results, 1)
 
     #
     # Concurrency tests
-- 
cgit v1.2.3


From 0e954f3328b7b8c31c88e0bee796230e87bca829 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sun, 20 Sep 2015 17:47:07 -0300
Subject: [feat] adds cache expiration

Now each backend object will be retrieved from cache for sync.py and
values will live for 3600 by default. That is changed via parameter if
needed.
---
 common/src/leap/soledad/common/couch.py   | 2 ++
 server/src/leap/soledad/server/caching.py | 4 ++--
 server/src/leap/soledad/server/sync.py    | 5 ++++-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 83f542ab..809af05d 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -527,6 +527,7 @@ class CouchDatabase(CommonBackend):
         try:
             # grab replica_uid from server
             doc = self._database['u1db_config']
+            self.cache[self._url] = doc
             self._real_replica_uid = doc['replica_uid']
             return self._real_replica_uid
         except ResourceNotFound:
@@ -1154,6 +1155,7 @@ class CouchDatabase(CommonBackend):
         doc['generation'] = other_generation
         doc['transaction_id'] = other_transaction_id
         self._database.save(doc)
+        self.cache[other_replica_uid] = (other_generation, other_transaction_id)
 
     def _force_doc_sync_conflict(self, doc):
         """
diff --git a/server/src/leap/soledad/server/caching.py b/server/src/leap/soledad/server/caching.py
index cd5d8dd4..9a049a39 100644
--- a/server/src/leap/soledad/server/caching.py
+++ b/server/src/leap/soledad/server/caching.py
@@ -28,5 +28,5 @@ def setup_caching():
 _cache_manager = setup_caching()
 
 
-def get_cache_for(key):
-    return _cache_manager.get_cache(key)
+def get_cache_for(key, expire=3600):
+    return _cache_manager.get_cache(key, expire=expire)
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index 262b6769..e4fd1260 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -185,12 +185,15 @@ class SyncResource(http_app.SyncResource):
         :type ensure: bool
         """
         # create or open the database
+        cache = get_cache_for('db-' + sync_id + self.dbname)
         if ensure:
             db, self.replica_uid = self.state.ensure_database(self.dbname)
+        elif cache and 'instance' in cache:
+            db = cache['instance']
         else:
             db = self.state.open_database(self.dbname)
-        cache = get_cache_for('db-' + sync_id + db.replica_uid)
         db._cache = cache
+        cache['instance'] = db
         # validate the information the client has about server replica
         db.validate_gen_and_trans_id(
             last_known_generation, last_known_trans_id)
-- 
cgit v1.2.3


From cb7aa314ad4d47e9f32e9e111ec13976978ed02d Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sun, 20 Sep 2015 19:28:22 -0300
Subject: [feat] enable delayed commits

We use CouchDB with single doc read/write. Following this documentation
about performance, we should get more performance by enabling couch to
delay and commit later.
See: http://guide.couchdb.org/draft/performance.html#single
---
 common/src/leap/soledad/common/couch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 809af05d..014b3f38 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -357,7 +357,7 @@ def couch_server(url):
     :type url: str
     """
     session = Session(timeout=COUCH_TIMEOUT)
-    server = Server(url=url, session=session)
+    server = Server(url=url, full_commit=False, session=session)
     yield server
 
 
-- 
cgit v1.2.3


From df8c794395ee695ea1dfb30603a073b32b24ee58 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 21 Sep 2015 17:19:18 -0300
Subject: [refactor] init_caching instead of setting attr

As meskio found commented, setting this attribute directly is ugly,
CouchDatabase now has a init_caching method for setting up cache
instance.
---
 common/src/leap/soledad/common/couch.py | 9 +++++++++
 server/src/leap/soledad/server/sync.py  | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 014b3f38..00a17715 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -443,6 +443,15 @@ class CouchDatabase(CommonBackend):
         else:
             return {}
 
+    def init_caching(self, cache):
+        """
+        Start using cache by setting internal _cache attribute.
+
+        :param cache: the cache instance, anything that behaves like a dict
+        :type cache: dict
+        """
+        self._cache = cache
+
     def ensure_ddocs_on_db(self):
         """
         Ensure that the design documents used by the backend exist on the
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index e4fd1260..619be565 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -192,7 +192,7 @@ class SyncResource(http_app.SyncResource):
             db = cache['instance']
         else:
             db = self.state.open_database(self.dbname)
-        db._cache = cache
+        db.init_caching(cache)
         cache['instance'] = db
         # validate the information the client has about server replica
         db.validate_gen_and_trans_id(
-- 
cgit v1.2.3


From 5b20b469f22ab99533135beefd49129db49064e5 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 23 Sep 2015 15:43:06 -0300
Subject: [style] pep8

---
 common/src/leap/soledad/common/couch.py | 15 +++++++++------
 server/src/leap/soledad/server/state.py | 24 +++++++++++-------------
 2 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 00a17715..38041c09 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -719,7 +719,8 @@ class CouchDatabase(CommonBackend):
             return None
         return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts)
 
-    def __parse_doc_from_couch(self, result, doc_id, check_for_conflicts=False):
+    def __parse_doc_from_couch(self, result, doc_id,
+                               check_for_conflicts=False):
         # restrict to u1db documents
         if 'u1db_rev' not in result:
             return None
@@ -1155,7 +1156,8 @@ class CouchDatabase(CommonBackend):
         :param sync_id: The id of the current sync session.
         :type sync_id: str
         """
-        self.cache[other_replica_uid] = (other_generation, other_transaction_id)
+        self.cache[other_replica_uid] = (other_generation,
+                                         other_transaction_id)
         doc_id = 'u1db_sync_%s' % other_replica_uid
         try:
             doc = self._database[doc_id]
@@ -1164,7 +1166,6 @@ class CouchDatabase(CommonBackend):
         doc['generation'] = other_generation
         doc['transaction_id'] = other_transaction_id
         self._database.save(doc)
-        self.cache[other_replica_uid] = (other_generation, other_transaction_id)
 
     def _force_doc_sync_conflict(self, doc):
         """
@@ -1175,7 +1176,8 @@ class CouchDatabase(CommonBackend):
         """
         my_doc = self._get_doc(doc.doc_id)
         self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev))
-        doc.add_conflict(self._factory(doc.doc_id, my_doc.rev, my_doc.get_json()))
+        doc.add_conflict(self._factory(doc.doc_id, my_doc.rev,
+                                       my_doc.get_json()))
         doc.has_conflicts = True
         self._put_doc(my_doc, doc)
 
@@ -1293,8 +1295,9 @@ class CouchDatabase(CommonBackend):
         my_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
         if my_doc:
             doc.set_conflicts(my_doc.get_conflicts())
-        return CommonBackend._put_doc_if_newer(self, doc, save_conflict, replica_uid,
-                                               replica_gen, replica_trans_id)
+        return CommonBackend._put_doc_if_newer(self, doc, save_conflict,
+                                               replica_uid, replica_gen,
+                                               replica_trans_id)
 
     def _put_and_update_indexes(self, cur_doc, doc):
         self._put_doc(cur_doc, doc)
diff --git a/server/src/leap/soledad/server/state.py b/server/src/leap/soledad/server/state.py
index d1225170..f269b77e 100644
--- a/server/src/leap/soledad/server/state.py
+++ b/server/src/leap/soledad/server/state.py
@@ -24,7 +24,8 @@ class ServerSyncState(object):
     """
     The state of one sync session, as stored on backend server.
 
-    On server side, the ongoing syncs metadata is maintained in a caching layer.
+    On server side, the ongoing syncs metadata is maintained in
+    a caching layer.
     """
 
     def __init__(self, source_replica_uid, sync_id):
@@ -38,7 +39,8 @@ class ServerSyncState(object):
         """
         self._source_replica_uid = source_replica_uid
         self._sync_id = sync_id
-        self._storage = caching.get_cache_for(source_replica_uid + sync_id)
+        caching_key = source_replica_uid + sync_id
+        self._storage = caching.get_cache_for(caching_key)
 
     def _put_dict_info(self, key, value):
         """
@@ -61,7 +63,7 @@ class ServerSyncState(object):
 
         :param seen_id: The doc_id of a document seen during sync.
         :type seen_id: str
-        :param gen: The corresponding db generation for that document.
+        :param gen: The corresponding db generation.
         :type gen: int
         """
         self._put_dict_info(
@@ -113,12 +115,9 @@ class ServerSyncState(object):
                  server.
         :rtype: tuple
         """
-        info = self._storage.get('changes_to_return') if 'changes_to_return' in self._storage else None
-        gen = None
-        trans_id = None
-        number_of_changes = None
-        if info:
-            info = info[0]
+        gen = trans_id = number_of_changes = None
+        if 'changes_to_return' in self._storage:
+            info = self._storage.get('changes_to_return')[0]
             gen = info['gen']
             trans_id = info['trans_id']
             number_of_changes = len(info['changes_to_return'])
@@ -132,12 +131,11 @@ class ServerSyncState(object):
                          received during the current sync process.
         :type received: int
         """
-        info = self._storage.get('changes_to_return') if 'changes_to_return' in self._storage else None
         gen = trans_id = next_change_to_return = None
-        if info:
-            info = info[0]
+        if 'changes_to_return' in self._storage:
+            info = self._storage.get('changes_to_return')[0]
             gen = info['gen']
             trans_id = info['trans_id']
             if received < len(info['changes_to_return']):
-                next_change_to_return = tuple(info['changes_to_return'][received])
+                next_change_to_return = (info['changes_to_return'][received])
         return gen, trans_id, next_change_to_return
-- 
cgit v1.2.3


From 4113fffc993b715629b484fe8f5a06d7efb9f644 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 15 Sep 2015 19:51:49 -0300
Subject: [feat] Validate and execute commands by subprocess

This commit adds a way to validate and execute commands using an
argument validator. Commands are executed via subprocess.
---
 common/src/leap/soledad/common/command.py | 54 +++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 common/src/leap/soledad/common/command.py

diff --git a/common/src/leap/soledad/common/command.py b/common/src/leap/soledad/common/command.py
new file mode 100644
index 00000000..978cec91
--- /dev/null
+++ b/common/src/leap/soledad/common/command.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# command.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Utility to sanitize and run shell commands.
+"""
+
+
+import subprocess
+
+
+def exec_validated_cmd(cmd, args, validator=None):
+    """
+    Executes cmd, validating args with validator.
+
+    :param cmd: command.
+    :type dbname: str
+    :param args: arguments.
+    :type args: str
+    :param validator: optional function to validate args
+    :type validator: function
+
+    :return: exit code and stdout or stderr (if code != 0)
+    :rtype: (int, str)
+    """
+    if validator and not validator(args):
+        return 1, "invalid argument"
+    command = cmd.split(' ')
+    command.append(args)
+    try:
+        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    except OSError, e:
+        return 1, e
+    (out, err) = process.communicate()
+    code = process.wait()
+    if code is not 0:
+        return code, err
+    else:
+        return code, out
-- 
cgit v1.2.3


From 66baa09a1760e911a6e58c87bce4f0062bd95805 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 15 Sep 2015 19:53:25 -0300
Subject: [tests] Tests for command validation and execution

Checks if arguments validation occurs properly and command execution
brings back status code and stdout or stderr on some scenarios.
---
 .../src/leap/soledad/common/tests/test_command.py  | 53 ++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 common/src/leap/soledad/common/tests/test_command.py

diff --git a/common/src/leap/soledad/common/tests/test_command.py b/common/src/leap/soledad/common/tests/test_command.py
new file mode 100644
index 00000000..af4903eb
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/test_command.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# test_command.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Tests for command execution using a validator function for arguments.
+"""
+from twisted.trial import unittest
+from leap.soledad.common.command import exec_validated_cmd
+
+
+class ExecuteValidatedCommandTest(unittest.TestCase):
+
+    def test_argument_validation(self):
+        validator = lambda arg: True if arg is 'valid' else False
+        status, out = exec_validated_cmd("command", "invalid arg", validator)
+        self.assertEquals(status, 1)
+        self.assertEquals(out, "invalid argument")
+        status, out = exec_validated_cmd("echo", "valid", validator)
+        self.assertEquals(status, 0)
+        self.assertEquals(out, "valid\n")
+
+    def test_return_status_code_success(self):
+        status, out = exec_validated_cmd("echo", "arg")
+        self.assertEquals(status, 0)
+        self.assertEquals(out, "arg\n")
+
+    def test_handle_command_with_spaces(self):
+        status, out = exec_validated_cmd("echo I am", "an argument")
+        self.assertEquals(status, 0, out)
+        self.assertEquals(out, "I am an argument\n")
+
+    def test_handle_oserror_on_invalid_command(self):
+        status, out = exec_validated_cmd("inexistent command with", "args")
+        self.assertEquals(status, 1)
+        self.assertIn("No such file or directory", out)
+
+    def test_return_status_code_number_on_failure(self):
+        status, out = exec_validated_cmd("ls", "user-bebacafe")
+        self.assertEquals(status, 2)
+        self.assertIn('ls: cannot access user-bebacafe: No such file or directory\n', out)
-- 
cgit v1.2.3


From eb6b66da6aa81ade4e61ef153ebbe8fba78cd335 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 15 Sep 2015 19:54:34 -0300
Subject: [feat] ensure_database is now able to call a cmd

If CouchServerState is created with a create_cmd parameter, it can now
use this parameter to invoke a command to create databases. A validator
for database name is also used to ensure that command injection is not
possible if user manages to manipulate database name argument.
---
 common/src/leap/soledad/common/couch.py | 46 +++++++++++++++++++++++++--------
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 38041c09..cd5185eb 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -60,6 +60,7 @@ from u1db.remote.server_state import ServerState
 
 
 from leap.soledad.common import ddocs, errors
+from leap.soledad.common.command import exec_validated_cmd
 from leap.soledad.common.document import SoledadDocument
 
 
@@ -1374,13 +1375,29 @@ class CouchSyncTarget(CommonSyncTarget):
             source_replica_transaction_id)
 
 
+DB_NAME_MASK = "^user-[a-f0-9]+$"
+
+
+def is_db_name_valid(name):
+    """
+    Validate a user database using DB_NAME_MASK.
+
+    :param name: database name.
+    :type name: str
+
+    :return: boolean for name vailidity
+    :rtype: bool
+    """
+    return re.match(DB_NAME_MASK, name) is not None
+
+
 class CouchServerState(ServerState):
 
     """
     Inteface of the WSGI server with the CouchDB backend.
     """
 
-    def __init__(self, couch_url):
+    def __init__(self, couch_url, create_cmd=None):
         """
         Initialize the couch server state.
 
@@ -1388,6 +1405,7 @@ class CouchServerState(ServerState):
         :type couch_url: str
         """
         self.couch_url = couch_url
+        self.create_cmd = create_cmd
 
     def open_database(self, dbname):
         """
@@ -1409,20 +1427,26 @@ class CouchServerState(ServerState):
         """
         Ensure couch database exists.
 
-        Usually, this method is used by the server to ensure the existence of
-        a database. In our setup, the Soledad user that accesses the underlying
-        couch server should never have permission to create (or delete)
-        databases. But, in case it ever does, by raising an exception here we
-        have one more guarantee that no modified client will be able to
-        enforce creation of a database when syncing.
-
         :param dbname: The name of the database to ensure.
         :type dbname: str
 
-        :raise Unauthorized: Always, because Soledad server is not allowed to
-                             create databases.
+        :raise Unauthorized: If disabled or other error was raised.
+
+        :return: The CouchDatabase object and its replica_uid.
+        :rtype: (CouchDatabase, str)
         """
-        raise Unauthorized()
+        if not self.create_cmd:
+            raise Unauthorized()
+        else:
+            code, out = exec_validated_cmd(self.create_cmd, dbname,
+                                           validator=is_db_name_valid)
+            if code is not 0:
+                logger.error("""
+                    Error while creating database (%s) with (%s) command.
+                    Output: %s
+                    Exit code: %d
+                    """ % (dbname, self.create_cmd, out, code))
+                raise Unauthorized()
 
     def delete_database(self, dbname):
         """
-- 
cgit v1.2.3


From 7591c95951e4618f7775c52340f4d170a1bdd961 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 15 Sep 2015 19:56:43 -0300
Subject: [tests] CouchServerState tests for ensure_database

Tests that Unauthorized is raised in any failure scenario, leaving user
blind for tips on what happened during execution. This should lower
chances of information disclosure on execution failure.
---
 common/src/leap/soledad/common/tests/test_couch.py | 25 ++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index c8d13667..d0a9dc3c 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -28,6 +28,7 @@ from couchdb.client import Server
 from uuid import uuid4
 
 from testscenarios import TestWithScenarios
+from twisted.trial import unittest
 
 from u1db import errors as u1db_errors
 from u1db import SyncTarget
@@ -1498,3 +1499,27 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
             self.db._get_transaction_log)
         self.create_db(ensure=True, dbname=self.db._dbname)
         self.db._get_transaction_log()
+
+
+class DatabaseNameValidationTest(unittest.TestCase):
+
+    def test_database_name_validation(self):
+        self.assertFalse(couch.is_db_name_valid("user-deadbeef | cat /secret"))
+        self.assertTrue(couch.is_db_name_valid("user-cafe1337"))
+
+
+class CommandBasedDBCreationTest(unittest.TestCase):
+
+    def test_ensure_db_using_custom_command(self):
+        state = couch.CouchServerState("url", create_cmd="echo")
+        state.ensure_database("user-1337")  # works
+
+    def test_raises_unauthorized_on_failure(self):
+        state = couch.CouchServerState("url", create_cmd="inexistent")
+        self.assertRaises(u1db_errors.Unauthorized,
+                          state.ensure_database, "user-1337")
+
+    def test_raises_unauthorized_by_default(self):
+        state = couch.CouchServerState("url")
+        self.assertRaises(u1db_errors.Unauthorized,
+                          state.ensure_database, "user-1337")
-- 
cgit v1.2.3


From e76ce03ad29b5582c4763d61a5acaed1aeba87e5 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 15 Sep 2015 19:59:14 -0300
Subject: [feat] conf for enabling db creation via custom sh

We can now use a custom script to create databases by setting a
parameter 'create_cmd' on soledad configuration.
This will set CouchServerState to use it on ensure_database.
---
 server/src/leap/soledad/server/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 1b795016..bb1c6db0 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -285,6 +285,7 @@ def load_configuration(file_path):
     """
     conf = {
         'couch_url': 'http://localhost:5984',
+        'create_cmd': None
     }
     config = configparser.ConfigParser()
     config.read(file_path)
@@ -303,7 +304,7 @@ def load_configuration(file_path):
 
 def application(environ, start_response):
     conf = load_configuration('/etc/leap/soledad-server.conf')
-    state = CouchServerState(conf['couch_url'])
+    state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'])
     # WSGI application that may be used by `twistd -web`
     application = GzipMiddleware(
         SoledadTokenAuthMiddleware(SoledadApp(state)))
-- 
cgit v1.2.3


From f7ff2e014e25b5c201f4e6209549518b53fc36b2 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 15 Sep 2015 20:12:01 -0300
Subject: [bug] ensure_database returns db and replica_uid

ensure database needs to return a db and its replica_uid. Updated tests,
doc and code to reflect that.
---
 common/src/leap/soledad/common/couch.py            | 2 ++
 common/src/leap/soledad/common/tests/test_couch.py | 8 +++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index cd5185eb..36f6239e 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -1447,6 +1447,8 @@ class CouchServerState(ServerState):
                     Exit code: %d
                     """ % (dbname, self.create_cmd, out, code))
                 raise Unauthorized()
+        db = self.open_database(dbname)
+        return db, db.replica_uid
 
     def delete_database(self, dbname):
         """
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index d0a9dc3c..a56cea21 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -29,6 +29,7 @@ from uuid import uuid4
 
 from testscenarios import TestWithScenarios
 from twisted.trial import unittest
+from mock import Mock
 
 from u1db import errors as u1db_errors
 from u1db import SyncTarget
@@ -1512,7 +1513,12 @@ class CommandBasedDBCreationTest(unittest.TestCase):
 
     def test_ensure_db_using_custom_command(self):
         state = couch.CouchServerState("url", create_cmd="echo")
-        state.ensure_database("user-1337")  # works
+        mock_db = Mock()
+        mock_db.replica_uid = 'replica_uid'
+        state.open_database = Mock(return_value=mock_db)
+        db, replica_uid = state.ensure_database("user-1337")  # works
+        self.assertEquals(mock_db, db)
+        self.assertEquals(mock_db.replica_uid, replica_uid)
 
     def test_raises_unauthorized_on_failure(self):
         state = couch.CouchServerState("url", create_cmd="inexistent")
-- 
cgit v1.2.3


From b065492f35006c3d108965b2b50144e080fbe678 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Thu, 17 Sep 2015 18:30:07 -0300
Subject: [feat] script for user db creation

Added a simple script for user db creation and design docs creation.
It uses a netrc from /etc/couchdb/couchdb-admin.netrc and same validator
used on couch.py for database names.
---
 server/pkg/create-user-db | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100755 server/pkg/create-user-db

diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
new file mode 100755
index 00000000..edcb8a82
--- /dev/null
+++ b/server/pkg/create-user-db
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+import os
+import sys
+import netrc
+import argparse
+from leap.soledad.common.couch import CouchDatabase
+from leap.soledad.common.couch import is_db_name_valid
+
+
+description = """
+Creates a user database.
+This is meant to be used by Soledad Server.
+"""
+parser = argparse.ArgumentParser(description=description)
+parser.add_argument('dbname', metavar='user-d34db33f', type=str,
+               help='database name on the format user-{uuid4}')
+NETRC_PATH = '/etc/couchdb/couchdb-admin.netrc'
+
+
+def url_for_db(dbname):
+    if not os.path.exists(NETRC_PATH):
+        print ('netrc not found in %s' % NETRC_PATH)
+        sys.exit(1)
+    parsed_netrc = netrc.netrc(NETRC_PATH)
+    host, (login, _, password) = parsed_netrc.hosts.items()[0]
+    url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % {
+           'login':login,
+           'password':password,
+           'host':host,
+           'dbname':dbname})
+    return url
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    if not is_db_name_valid(args.dbname):
+        print ("Invalid name! %s" % args.dbname)
+        sys.exit(1)
+    url = url_for_db(args.dbname)
+    db = CouchDatabase.open_database(url=url, create=True,
+                                     replica_uid=None, ensure_ddocs=True)
+    print ('success! Created %s, replica_uid: %s' % (db._dbname, db.replica_uid))
-- 
cgit v1.2.3


From a660f60b9644836b0dbdf54cd04b15f4d4654d0f Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 18 Sep 2015 17:30:19 -0300
Subject: [feat] ensure security document

Beyond ensuring ddocs, it is also necessary to ensure _security doc
presence while creating a database.
This document will tell couchdb to grant access to 'soledad' user as a
member role and no one as admin.
---
 common/src/leap/soledad/common/couch.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 36f6239e..d9ed5026 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -435,6 +435,7 @@ class CouchDatabase(CommonBackend):
             self._set_replica_uid(replica_uid)
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
+            self.ensure_security()
         self._cache = None
 
     @property
@@ -467,6 +468,16 @@ class CouchDatabase(CommonBackend):
                         getattr(ddocs, ddoc_name)))
                 self._database.save(ddoc)
 
+    def ensure_security(self):
+        """
+        Make sure that only soledad user is able to access this database as
+        a member.
+        """
+        security = self._database.security
+        security['members'] = {'names': ['soledad'], 'roles': []}
+        security['admins'] = {'names': [], 'roles': []}
+        self._database.security = security
+
     def get_sync_target(self):
         """
         Return a SyncTarget object, for another u1db to synchronize with.
-- 
cgit v1.2.3


From 7b1591e3d561aa6525318235e702eebeb6708560 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 18 Sep 2015 17:31:37 -0300
Subject: [tests] tests for ensure_security

As the other tests does. Make sure that a fresh database gets proper
security doc after calling ensure_security method.
---
 common/src/leap/soledad/common/tests/test_couch.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index a56cea21..3622bb56 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -1501,6 +1501,20 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         self.create_db(ensure=True, dbname=self.db._dbname)
         self.db._get_transaction_log()
 
+    def test_ensure_security_doc(self):
+        """
+        Ensure_security creates a _security ddoc to ensure that only soledad
+        will have member access to a db.
+        """
+        self.create_db(ensure=False)
+        self.assertFalse(self.db._database.security)
+        self.db.ensure_security()
+        security_ddoc = self.db._database.security
+        self.assertIn('admins', security_ddoc)
+        self.assertFalse(security_ddoc['admins']['names'])
+        self.assertIn('members', security_ddoc)
+        self.assertIn('soledad', security_ddoc['members']['names'])
+
 
 class DatabaseNameValidationTest(unittest.TestCase):
 
-- 
cgit v1.2.3


From c4c280e12bf70dd41183d2d2ffbba200a45d0247 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sat, 19 Sep 2015 14:25:30 -0300
Subject: [feat] handle DatabaseDoesNotExist during sync

This error raises while getting info on target (or server) replica. On
previous implementation there was nothing to do here, but now that we
have db creation in place this error should be handled just like u1db
original implementation.
The reason is that db creation occurs during sync exchange, but before
this we try to ask for info and the code that checks for info raises an
error that will be used to signal the client that a database will be
created and that it must save the replica uid returned by server, after
sync exchange (where we send/fetch documents).
---
 client/src/leap/soledad/client/sync.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py
index 110baa0a..225d3e2d 100644
--- a/client/src/leap/soledad/client/sync.py
+++ b/client/src/leap/soledad/client/sync.py
@@ -69,9 +69,15 @@ class SoledadSynchronizer(Synchronizer):
         # get target identifier, its current generation,
         # and its last-seen database generation for this source
         ensure_callback = None
-        (self.target_replica_uid, target_gen, target_trans_id,
-         target_my_gen, target_my_trans_id) = yield \
-            sync_target.get_sync_info(self.source._replica_uid)
+        try:
+            (self.target_replica_uid, target_gen, target_trans_id,
+             target_my_gen, target_my_trans_id) = yield \
+                sync_target.get_sync_info(self.source._replica_uid)
+        except errors.DatabaseDoesNotExist:
+            logger.debug("Database isn't ready on server. Will be created.")
+            self.target_replica_uid = None
+            target_gen, target_trans_id = 0, ''
+            target_my_gen, target_my_trans_id = 0, ''
 
         logger.debug(
             "Soledad target sync info:\n"
-- 
cgit v1.2.3


From de0cf00b4412e253a481ff19803bab66ffc4443e Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Thu, 24 Sep 2015 21:57:26 -0300
Subject: [refactor] kaliy's review and pep8 fixes

README with information about latest change, missing docs and licenses,
variable naming and pep8.
---
 README.rst                                         | 33 ++++++++++++++++++++++
 common/changes/create_db_cmd                       |  2 ++
 common/src/leap/soledad/common/command.py          | 17 +++++------
 common/src/leap/soledad/common/couch.py            | 19 +++++++------
 .../src/leap/soledad/common/tests/test_command.py  |  4 ++-
 common/src/leap/soledad/common/tests/test_couch.py |  4 +--
 server/changes/create_db_cmd                       |  3 ++
 server/pkg/create-user-db                          | 16 +++++++++++
 8 files changed, 79 insertions(+), 19 deletions(-)
 create mode 100644 common/changes/create_db_cmd
 create mode 100644 server/changes/create_db_cmd

diff --git a/README.rst b/README.rst
index b98eec06..a2755f92 100644
--- a/README.rst
+++ b/README.rst
@@ -55,3 +55,36 @@ to run tests in development mode you must do the following::
 
 Note that to run CouchDB tests, be sure you have ``CouchDB`` installed on your
 system.
+
+
+Privileges
+-----
+In order to prevent privilege escalation, Soledad should not be run as a
+database administrator. This implies the following side effects:
+
+-----------------
+Database creation:
+-----------------
+Can be done via a script located in ``server/pkg/create-user-db``
+It reads a netrc file that should be placed on
+``/etc/couchdb/couchdb-admin.netrc``.
+That file holds the admin credentials in netrc format and should be accessible
+only by 'soledad-admin' user.
+
+The debian package will do the following in order to automate this:
+
+* create a user ``soledad-admin``
+* make this script available as ``create-user-db`` in ``/usr/bin``
+* grant restricted sudo access, that only enables user ``soledad`` to call this
+  exact command via ``soledad-admin`` user.
+
+The server side process, configured via ``/etc/leap/soledad-server.conf``, will
+then use a parameter called 'create_cmd' to know which command is used to
+allocate new databases. All steps of creation process is then handled
+automatically by the server, following the same logic as u1db server.
+
+------------------
+Database deletion:
+------------------
+No code at all handles this and privilege to do so needs to be removed as
+explained before. This can be automated via a simple cron job.
diff --git a/common/changes/create_db_cmd b/common/changes/create_db_cmd
new file mode 100644
index 00000000..00bbdf71
--- /dev/null
+++ b/common/changes/create_db_cmd
@@ -0,0 +1,2 @@
+  o Add a sanitized command executor for database creation and re-enable
+  user database creation on CouchServerState via command line.
diff --git a/common/src/leap/soledad/common/command.py b/common/src/leap/soledad/common/command.py
index 978cec91..811bf135 100644
--- a/common/src/leap/soledad/common/command.py
+++ b/common/src/leap/soledad/common/command.py
@@ -24,26 +24,27 @@ Utility to sanitize and run shell commands.
 import subprocess
 
 
-def exec_validated_cmd(cmd, args, validator=None):
+def exec_validated_cmd(cmd, argument, validator=None):
     """
-    Executes cmd, validating args with validator.
+    Executes cmd, validating argument with a validator function.
 
     :param cmd: command.
     :type dbname: str
-    :param args: arguments.
-    :type args: str
-    :param validator: optional function to validate args
+    :param argument: argument.
+    :type argument: str
+    :param validator: optional function to validate argument
     :type validator: function
 
     :return: exit code and stdout or stderr (if code != 0)
     :rtype: (int, str)
     """
-    if validator and not validator(args):
+    if validator and not validator(argument):
         return 1, "invalid argument"
     command = cmd.split(' ')
-    command.append(args)
+    command.append(argument)
     try:
-        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        process = subprocess.Popen(command, stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
     except OSError, e:
         return 1, e
     (out, err) = process.communicate()
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index d9ed5026..4c5f6400 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -435,7 +435,7 @@ class CouchDatabase(CommonBackend):
             self._set_replica_uid(replica_uid)
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
-            self.ensure_security()
+            self.ensure_security_ddoc()
         self._cache = None
 
     @property
@@ -468,10 +468,15 @@ class CouchDatabase(CommonBackend):
                         getattr(ddocs, ddoc_name)))
                 self._database.save(ddoc)
 
-    def ensure_security(self):
+    def ensure_security_ddoc(self):
         """
         Make sure that only soledad user is able to access this database as
-        a member.
+        an unprivileged member, meaning that administration access will
+        be forbidden even inside an user database.
+        The goal is to make sure that only the lowest access level is given
+        to the unprivileged CouchDB user set on the server process.
+        This is achieved by creating a _security design document, see:
+        http://docs.couchdb.org/en/latest/api/database/security.html
         """
         security = self._database.security
         security['members'] = {'names': ['soledad'], 'roles': []}
@@ -1386,12 +1391,9 @@ class CouchSyncTarget(CommonSyncTarget):
             source_replica_transaction_id)
 
 
-DB_NAME_MASK = "^user-[a-f0-9]+$"
-
-
 def is_db_name_valid(name):
     """
-    Validate a user database using DB_NAME_MASK.
+    Validate a user database using a regular expression.
 
     :param name: database name.
     :type name: str
@@ -1399,7 +1401,8 @@ def is_db_name_valid(name):
     :return: boolean for name vailidity
     :rtype: bool
     """
-    return re.match(DB_NAME_MASK, name) is not None
+    db_name_regex = "^user-[a-f0-9]+$"
+    return re.match(db_name_regex, name) is not None
 
 
 class CouchServerState(ServerState):
diff --git a/common/src/leap/soledad/common/tests/test_command.py b/common/src/leap/soledad/common/tests/test_command.py
index af4903eb..420f91ae 100644
--- a/common/src/leap/soledad/common/tests/test_command.py
+++ b/common/src/leap/soledad/common/tests/test_command.py
@@ -50,4 +50,6 @@ class ExecuteValidatedCommandTest(unittest.TestCase):
     def test_return_status_code_number_on_failure(self):
         status, out = exec_validated_cmd("ls", "user-bebacafe")
         self.assertEquals(status, 2)
-        self.assertIn('ls: cannot access user-bebacafe: No such file or directory\n', out)
+        self.assertIn(
+            'ls: cannot access user-bebacafe: No such file or directory\n',
+            out)
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 3622bb56..d1a07a3a 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -1504,11 +1504,11 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
     def test_ensure_security_doc(self):
         """
         Ensure_security creates a _security ddoc to ensure that only soledad
-        will have member access to a db.
+        will have the lowest privileged access to an user db.
         """
         self.create_db(ensure=False)
         self.assertFalse(self.db._database.security)
-        self.db.ensure_security()
+        self.db.ensure_security_ddoc()
         security_ddoc = self.db._database.security
         self.assertIn('admins', security_ddoc)
         self.assertFalse(security_ddoc['admins']['names'])
diff --git a/server/changes/create_db_cmd b/server/changes/create_db_cmd
new file mode 100644
index 00000000..cee0a935
--- /dev/null
+++ b/server/changes/create_db_cmd
@@ -0,0 +1,3 @@
+  o Adds a new config parameter 'create_cmd', which allows sysadmin to specify
+  which command will create a database. That command was added in
+  pkg/create-user-db and debian package automates steps needed for sudo access.
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index edcb8a82..dd68f792 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -1,4 +1,20 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# create-user-db
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
 import os
 import sys
 import netrc
-- 
cgit v1.2.3


From 01314341c8ef1e947b8f921ba023ac67d89a9ce7 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 28 Sep 2015 15:43:49 -0300
Subject: [test] remove old mocks

Those hardcoded mocks are leaking into other tests and are unnecessary.
---
 .../soledad/common/tests/test_couch_operations_atomicity.py  |  6 ------
 common/src/leap/soledad/common/tests/test_server.py          | 12 ------------
 2 files changed, 18 deletions(-)

diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
index 3e8e8cce..507f2984 100644
--- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
+++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
@@ -35,17 +35,11 @@ from leap.soledad.common.tests.util import (
 )
 from leap.soledad.common.tests.test_couch import CouchDBTestCase
 from leap.soledad.common.tests.u1db_tests import TestCaseWithServer
-from leap.soledad.common.tests.test_server import _couch_ensure_database
 
 
 REPEAT_TIMES = 20
 
 
-# monkey path CouchServerState so it can ensure databases.
-
-CouchServerState.ensure_database = _couch_ensure_database
-
-
 class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
 
     @staticmethod
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index f512d6c1..19d2907d 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -46,18 +46,6 @@ from leap.soledad.server import LockResource
 from leap.soledad.server.auth import URLToAuthorization
 
 
-# monkey path CouchServerState so it can ensure databases.
-
-def _couch_ensure_database(self, dbname):
-    db = CouchDatabase.open_database(
-        self.couch_url + '/' + dbname,
-        create=True,
-        ensure_ddocs=True)
-    return db, db._replica_uid
-
-CouchServerState.ensure_database = _couch_ensure_database
-
-
 class ServerAuthorizationTestCase(BaseSoledadTest):
 
     """
-- 
cgit v1.2.3


From 6427b73247e327c27d5f8e5c2036281fb77205b2 Mon Sep 17 00:00:00 2001
From: Gislene Pereira <gislene01@gmail.com>
Date: Mon, 28 Sep 2015 16:04:46 -0300
Subject: [test] Making test_command pass on Mac OS X.

---
 common/src/leap/soledad/common/tests/test_command.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/common/src/leap/soledad/common/tests/test_command.py b/common/src/leap/soledad/common/tests/test_command.py
index 420f91ae..c386bdd2 100644
--- a/common/src/leap/soledad/common/tests/test_command.py
+++ b/common/src/leap/soledad/common/tests/test_command.py
@@ -49,7 +49,5 @@ class ExecuteValidatedCommandTest(unittest.TestCase):
 
     def test_return_status_code_number_on_failure(self):
         status, out = exec_validated_cmd("ls", "user-bebacafe")
-        self.assertEquals(status, 2)
-        self.assertIn(
-            'ls: cannot access user-bebacafe: No such file or directory\n',
-            out)
+        self.assertNotEquals(status, 0)
+        self.assertIn('No such file or directory\n', out)
-- 
cgit v1.2.3


From 3c7a41574ed1a97ae168bbbc50b127d17694734a Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 28 Sep 2015 16:35:19 -0300
Subject: [style] pep8

---
 server/pkg/create-user-db | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index dd68f792..1a7e77a7 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -29,7 +29,7 @@ This is meant to be used by Soledad Server.
 """
 parser = argparse.ArgumentParser(description=description)
 parser.add_argument('dbname', metavar='user-d34db33f', type=str,
-               help='database name on the format user-{uuid4}')
+                    help='database name on the format user-{uuid4}')
 NETRC_PATH = '/etc/couchdb/couchdb-admin.netrc'
 
 
@@ -40,10 +40,10 @@ def url_for_db(dbname):
     parsed_netrc = netrc.netrc(NETRC_PATH)
     host, (login, _, password) = parsed_netrc.hosts.items()[0]
     url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % {
-           'login':login,
-           'password':password,
-           'host':host,
-           'dbname':dbname})
+           'login': login,
+           'password': password,
+           'host': host,
+           'dbname': dbname})
     return url
 
 
@@ -55,4 +55,5 @@ if __name__ == '__main__':
     url = url_for_db(args.dbname)
     db = CouchDatabase.open_database(url=url, create=True,
                                      replica_uid=None, ensure_ddocs=True)
-    print ('success! Created %s, replica_uid: %s' % (db._dbname, db.replica_uid))
+    print ('success! Created %s, replica_uid: %s' %
+           (db._dbname, db.replica_uid))
-- 
cgit v1.2.3


From 9f7285403f6e012b8ea5406cf9a2a19e68412210 Mon Sep 17 00:00:00 2001
From: varac <varacanero@zeromail.org>
Date: Tue, 29 Sep 2015 15:39:25 +0200
Subject: [docs] added couchdb link to README

---
 README.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index a2755f92..36746c82 100644
--- a/README.rst
+++ b/README.rst
@@ -1,9 +1,9 @@
-Soledad 
+Soledad
 ==================================================================
 *Synchronization Of Locally Encrypted Data Among Devices*
 
 Soledad is the part of LEAP that allows application data to be
-securely shared among devices. It provides, to other parts of the 
+securely shared among devices. It provides, to other parts of the
 LEAP project, an API for data storage and sync.
 
 This software is under development.
@@ -53,7 +53,7 @@ to run tests in development mode you must do the following::
   scripts/develop_mode.sh
   ./run_tests.sh
 
-Note that to run CouchDB tests, be sure you have ``CouchDB`` installed on your
+Note that to run CouchDB tests, be sure you have [CouchDB](https://couchdb.apache.org/) installed on your
 system.
 
 
-- 
cgit v1.2.3


From 4964eed77748d65df30847a5fe0a608a163af7e0 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 30 Sep 2015 21:35:05 -0400
Subject: [docs] fix rst link markup

---
 README.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 36746c82..631f90bb 100644
--- a/README.rst
+++ b/README.rst
@@ -53,9 +53,10 @@ to run tests in development mode you must do the following::
   scripts/develop_mode.sh
   ./run_tests.sh
 
-Note that to run CouchDB tests, be sure you have [CouchDB](https://couchdb.apache.org/) installed on your
+Note that to run CouchDB tests, be sure you have `CouchDB`_ installed on your
 system.
 
+.. _`CouchDB`: https://couchdb.apache.org/
 
 Privileges
 -----
-- 
cgit v1.2.3


From 7a0dba8b7008aca8652f7b334352d9ca63cb2054 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 30 Sep 2015 15:27:21 -0300
Subject: [feat] read netrc path from configuration file

netrc file was hardcoded inside create-user-db. Now it reads the path
from /etc/leap/soledad-server.conf as done on server process.
The new configuration property is called 'admin_netrc'.
---
 server/changes/create_db_cmd               |  1 +
 server/pkg/create-user-db                  |  3 ++-
 server/src/leap/soledad/server/__init__.py | 11 ++++++-----
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/server/changes/create_db_cmd b/server/changes/create_db_cmd
index cee0a935..964a7906 100644
--- a/server/changes/create_db_cmd
+++ b/server/changes/create_db_cmd
@@ -1,3 +1,4 @@
   o Adds a new config parameter 'create_cmd', which allows sysadmin to specify
   which command will create a database. That command was added in
   pkg/create-user-db and debian package automates steps needed for sudo access.
+  o Read netrc path from configuration file for create-user-db command. 
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index 1a7e77a7..ae2f0f57 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -21,6 +21,7 @@ import netrc
 import argparse
 from leap.soledad.common.couch import CouchDatabase
 from leap.soledad.common.couch import is_db_name_valid
+from leap.soledad.server import load_configuration
 
 
 description = """
@@ -30,7 +31,7 @@ This is meant to be used by Soledad Server.
 parser = argparse.ArgumentParser(description=description)
 parser.add_argument('dbname', metavar='user-d34db33f', type=str,
                     help='database name on the format user-{uuid4}')
-NETRC_PATH = '/etc/couchdb/couchdb-admin.netrc'
+NETRC_PATH = load_configuration('/etc/leap/soledad-server.conf')['admin_netrc']
 
 
 def url_for_db(dbname):
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index bb1c6db0..1273d15c 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -283,19 +283,20 @@ def load_configuration(file_path):
     @return: A dictionary with the configuration.
     @rtype: dict
     """
-    conf = {
+    defaults = {
         'couch_url': 'http://localhost:5984',
-        'create_cmd': None
+        'create_cmd': None,
+        'admin_netrc': '/etc/couchdb/couchdb-admin.netrc',
     }
     config = configparser.ConfigParser()
     config.read(file_path)
     if 'soledad-server' in config:
-        for key in conf:
+        for key in defaults:
             if key in config['soledad-server']:
-                conf[key] = config['soledad-server'][key]
+                defaults[key] = config['soledad-server'][key]
     # TODO: implement basic parsing/sanitization of options comming from
     # config file.
-    return conf
+    return defaults
 
 
 # ----------------------------------------------------------------------------
-- 
cgit v1.2.3


From 234e5879188fe100bf99b9ef6f252abff06b4dad Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 2 Oct 2015 12:03:29 -0400
Subject: [bug] do not signal sync completion if failed

- Related: #7503
---
 client/changes/bug_7503-do-not-signal-sync-complete | 1 +
 client/src/leap/soledad/client/api.py               | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 client/changes/bug_7503-do-not-signal-sync-complete

diff --git a/client/changes/bug_7503-do-not-signal-sync-complete b/client/changes/bug_7503-do-not-signal-sync-complete
new file mode 100644
index 00000000..4cc361e0
--- /dev/null
+++ b/client/changes/bug_7503-do-not-signal-sync-complete
@@ -0,0 +1 @@
+o Do not signal sync completion if sync failed. Closes: #7503
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index a558addd..8c5f7f1b 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -723,7 +723,7 @@ class Soledad(object):
             return passthrough
 
         d.addCallbacks(_sync_callback, _sync_errback)
-        d.addBoth(_emit_done_data_sync)
+        d.addCallback(_emit_done_data_sync)
         return d
 
     @property
-- 
cgit v1.2.3


From 2121d21945ef7458eb57c3cab53a98b92fce1417 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 2 Oct 2015 11:54:54 -0400
Subject: [bug] increase http request timeout time to 90s

this is a workaroud to reduce the chances of failed sync due to
timeouts. this should be properly tackled by:

1. implementing proper cancellable for the sync operation.
2. implementing a retry count at the level of a single request, handled
internally by soledad.

in this way we can remove the retries logic from the soledadbootstrapper
in the bitmask client.

- Related: #7382
---
 client/src/leap/soledad/client/http_target/__init__.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py
index 7a5cea9f..498fb6e7 100644
--- a/client/src/leap/soledad/client/http_target/__init__.py
+++ b/client/src/leap/soledad/client/http_target/__init__.py
@@ -87,4 +87,8 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher):
         # asynchronous encryption/decryption attributes
         self._decryption_callback = None
         self._sync_decr_pool = None
-        self._http = HTTPClient(cert_file)
+
+        # XXX Increasing timeout of simple requests to avoid chances of hitting
+        # the duplicated syncing bug. This could be reduced to the 30s default
+        # after implementing Cancellable Sync. See #7382
+        self._http = HTTPClient(cert_file, timeout=90)
-- 
cgit v1.2.3


From a48b630ef48377b7307b59333f8eb02190786012 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 2 Oct 2015 17:37:52 -0300
Subject: [pkg] pin beaker and couchdb for wheezy

Wheezy has python-couchdb 0.8 and python-beaker 1.6.3.
Pinning them to avoid false positives on tests.
---
 server/pkg/requirements.pip | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip
index 3e1aa992..58834d0e 100644
--- a/server/pkg/requirements.pip
+++ b/server/pkg/requirements.pip
@@ -1,10 +1,11 @@
 configparser
-couchdb
 u1db
 routes
 PyOpenSSL
 twisted
-Beaker
+#pinned for wheezy compatibility
+Beaker==1.6.3 #wheezy
+couchdb==0.8 #wheezy
 
 # XXX -- fix me!
 # oauth is not strictly needed by us, but we need it until u1db adds it to its
-- 
cgit v1.2.3


From c7baad4f07ec8a44bd12113775c40974be3a4fb9 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 2 Oct 2015 17:44:11 -0300
Subject: [bug] Migrate back to python-couchdb 0.8

Wheezy is still at 0.8 and it is yet supported.
This commit changes all necessary calls from python-couchdb 1.0 back to
python-couchdb 0.8. We can migrate this back to simpler implementation
with python-couchdb 1.0 when support for wheezy is dropped.
---
 common/src/leap/soledad/common/couch.py            | 20 ++++++++++++++++----
 common/src/leap/soledad/common/tests/test_couch.py |  4 ++--
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 4c5f6400..10a1b277 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -461,7 +461,8 @@ class CouchDatabase(CommonBackend):
         """
         for ddoc_name in ['docs', 'syncs', 'transactions']:
             try:
-                self._database.info(ddoc_name)
+                self._database.resource('_design',
+                                        ddoc_name, '_info').get_json()
             except ResourceNotFound:
                 ddoc = json.loads(
                     binascii.a2b_base64(
@@ -478,10 +479,10 @@ class CouchDatabase(CommonBackend):
         This is achieved by creating a _security design document, see:
         http://docs.couchdb.org/en/latest/api/database/security.html
         """
-        security = self._database.security
+        security = self._database.resource.get_json('_security')[2]
         security['members'] = {'names': ['soledad'], 'roles': []}
         security['admins'] = {'names': [], 'roles': []}
-        self._database.security = security
+        self._database.resource.put_json('_security', body=security)
 
     def get_sync_target(self):
         """
@@ -888,7 +889,7 @@ class CouchDatabase(CommonBackend):
         try:
             resource = self._new_resource()
             resource.put_json(
-                doc.doc_id, body=buf.getvalue(), headers=envelope.headers)
+                doc.doc_id, body=str(buf.getvalue()), headers=envelope.headers)
         except ResourceConflict:
             raise RevisionConflict()
         if self.replica_uid + '_gen' in self.cache:
@@ -1337,6 +1338,17 @@ class CouchDatabase(CommonBackend):
                  in matching doc_ids order.
         :rtype: iterable
         """
+        # Workaround for:
+        #
+        #   http://bugs.python.org/issue7980
+        #   https://leap.se/code/issues/5449
+        #
+        # python-couchdb uses time.strptime, which is not thread safe. In
+        # order to avoid the problem described on the issues above, we preload
+        # strptime here by evaluating the conversion of an arbitrary date.
+        # This will not be needed when/if we switch from python-couchdb to
+        # paisley.
+        time.strptime('Mar 8 1917', '%b %d %Y')
         get_one = lambda doc_id: self._get_doc(doc_id, check_for_conflicts)
         docs = [THREAD_POOL.apply_async(get_one, [doc_id])
                 for doc_id in doc_ids]
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index d1a07a3a..b4797f5e 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -1507,9 +1507,9 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         will have the lowest privileged access to an user db.
         """
         self.create_db(ensure=False)
-        self.assertFalse(self.db._database.security)
+        self.assertFalse(self.db._database.resource.get_json('_security')[2])
         self.db.ensure_security_ddoc()
-        security_ddoc = self.db._database.security
+        security_ddoc = self.db._database.resource.get_json('_security')[2]
         self.assertIn('admins', security_ddoc)
         self.assertFalse(security_ddoc['admins']['names'])
         self.assertIn('members', security_ddoc)
-- 
cgit v1.2.3


From 147986547f400f79e8bc0d50af5ae7b5d2a140b4 Mon Sep 17 00:00:00 2001
From: varac <varacanero@zeromail.org>
Date: Mon, 5 Oct 2015 11:54:56 +0200
Subject: [feat] Move config dir to /etc/soledad

- Resolves: #7509
---
 README.rst                                                     | 2 +-
 scripts/db_access/reset_db.py                                  | 2 +-
 scripts/db_access/server_side_db.py                            | 4 ++--
 scripts/ddocs/update_design_docs.py                            | 2 +-
 scripts/profiling/doc_put_memory_usage/find_max_upload_size.py | 2 +-
 server/changes/change_soledad_configdir                        | 2 ++
 server/pkg/create-user-db                                      | 2 +-
 server/pkg/soledad-server                                      | 8 ++++----
 server/src/leap/soledad/server/__init__.py                     | 2 +-
 9 files changed, 14 insertions(+), 12 deletions(-)
 create mode 100644 server/changes/change_soledad_configdir

diff --git a/README.rst b/README.rst
index 631f90bb..815db808 100644
--- a/README.rst
+++ b/README.rst
@@ -79,7 +79,7 @@ The debian package will do the following in order to automate this:
 * grant restricted sudo access, that only enables user ``soledad`` to call this
   exact command via ``soledad-admin`` user.
 
-The server side process, configured via ``/etc/leap/soledad-server.conf``, will
+The server side process, configured via ``/etc/soledad/soledad-server.conf``, will
 then use a parameter called 'create_cmd' to know which command is used to
 allocate new databases. All steps of creation process is then handled
 automatically by the server, following the same logic as u1db server.
diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py
index 7c6d281b..c48b511e 100644
--- a/scripts/db_access/reset_db.py
+++ b/scripts/db_access/reset_db.py
@@ -63,7 +63,7 @@ def get_url(empty):
     if empty is False:
         # get couch url
         cp = ConfigParser()
-        cp.read('/etc/leap/soledad-server.conf')
+        cp.read('/etc/soledad/soledad-server.conf')
         url = cp.get('soledad-server', 'couch_url')
     else:
         with open('/etc/couchdb/couchdb.netrc') as f:
diff --git a/scripts/db_access/server_side_db.py b/scripts/db_access/server_side_db.py
index 18641a0f..fcdd14b6 100644
--- a/scripts/db_access/server_side_db.py
+++ b/scripts/db_access/server_side_db.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 
 # This script gives server-side access to one Soledad user database by using
-# the configuration stored in /etc/leap/soledad-server.conf.
+# the configuration stored in /etc/soledad/soledad-server.conf.
 #
 # Use it like this:
 # 
@@ -20,7 +20,7 @@ uuid = sys.argv[1]
 
 # get couch url
 cp = ConfigParser()
-cp.read('/etc/leap/soledad-server.conf')
+cp.read('/etc/soledad/soledad-server.conf')
 url = cp.get('soledad-server', 'couch_url')
 
 # access user db
diff --git a/scripts/ddocs/update_design_docs.py b/scripts/ddocs/update_design_docs.py
index 2e2fa8f0..281482b8 100644
--- a/scripts/ddocs/update_design_docs.py
+++ b/scripts/ddocs/update_design_docs.py
@@ -50,7 +50,7 @@ def _parse_args():
 def _get_url():
     # get couch url
     cp = ConfigParser()
-    cp.read('/etc/leap/soledad-server.conf')
+    cp.read('/etc/soledad/soledad-server.conf')
     url = urlparse(cp.get('soledad-server', 'couch_url'))
     # get admin password
     netloc = re.sub('^.*@', '', url.netloc)
diff --git a/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py
index 02c68015..1a603fd0 100755
--- a/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py
+++ b/scripts/profiling/doc_put_memory_usage/find_max_upload_size.py
@@ -30,7 +30,7 @@ from socket import error as socket_error
 from leap.soledad.common.couch import CouchDatabase
 
 
-SOLEDAD_CONFIG_FILE = '/etc/leap/soledad-server.conf'
+SOLEDAD_CONFIG_FILE = '/etc/soledad/soledad-server.conf'
 PREFIX = '/tmp/soledad_test'
 LOG_FORMAT = '%(asctime)s %(levelname)s %(message)s'
 RETRIES = 3  # number of times to retry uploading a document of a certain
diff --git a/server/changes/change_soledad_configdir b/server/changes/change_soledad_configdir
new file mode 100644
index 00000000..710b9ac8
--- /dev/null
+++ b/server/changes/change_soledad_configdir
@@ -0,0 +1,2 @@
+o Moves config directory from /etc/leap to /etc/soledad
+  resolves #7509
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index ae2f0f57..7eafc945 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -31,7 +31,7 @@ This is meant to be used by Soledad Server.
 parser = argparse.ArgumentParser(description=description)
 parser.add_argument('dbname', metavar='user-d34db33f', type=str,
                     help='database name on the format user-{uuid4}')
-NETRC_PATH = load_configuration('/etc/leap/soledad-server.conf')['admin_netrc']
+NETRC_PATH = load_configuration('/etc/soledad/soledad-server.conf')['admin_netrc']
 
 
 def url_for_db(dbname):
diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server
index 811ad55b..74ed122e 100644
--- a/server/pkg/soledad-server
+++ b/server/pkg/soledad-server
@@ -11,12 +11,12 @@
 
 PATH=/sbin:/bin:/usr/sbin:/usr/bin
 PIDFILE=/var/run/soledad.pid
-RUNDIR=/var/lib/soledad/
 OBJ=leap.soledad.server.application
 LOGFILE=/var/log/soledad.log
 HTTPS_PORT=2424
-CERT_PATH=/etc/leap/soledad-server.pem
-PRIVKEY_PATH=/etc/leap/soledad-server.key
+CONFDIR=/etc/soledad
+CERT_PATH="${CONFDIR}/soledad-server.pem"
+PRIVKEY_PATH="${CONFDIR}/soledad-server.key"
 TWISTD_PATH=/usr/bin/twistd
 HOME=/var/lib/soledad/
 SSL_METHOD=SSLv23_METHOD
@@ -25,7 +25,7 @@ GROUP=soledad
 
 [ -r /etc/default/soledad ] && . /etc/default/soledad
 
-test -r /etc/leap/ || exit 0
+test -r ${CONFDIR} || exit 0
 
 . /lib/lsb/init-functions
 
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 1273d15c..f64d07bf 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -304,7 +304,7 @@ def load_configuration(file_path):
 # ----------------------------------------------------------------------------
 
 def application(environ, start_response):
-    conf = load_configuration('/etc/leap/soledad-server.conf')
+    conf = load_configuration('/etc/soledad/soledad-server.conf')
     state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'])
     # WSGI application that may be used by `twistd -web`
     application = GzipMiddleware(
-- 
cgit v1.2.3


From dccaf79789b5f52848cfedd82ca4a2999de1432c Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 5 Oct 2015 12:19:12 -0300
Subject: [bug] fix leap mx by default false on ensure_ddocs

ensure_ddocs is a privileged operation. The code was defaulting to True,
which caused unprivileged code to fail. This commit changes it to False,
forcing you to check your privileges and declare a new argument when
calling in order to ensure that this behavior is only supposed to happen
on privileged parts.
---
 common/src/leap/soledad/common/couch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 10a1b277..3dee1473 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -404,7 +404,7 @@ class CouchDatabase(CommonBackend):
         return cls(
             url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs)
 
-    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True):
+    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=False):
         """
         Create a new Couch data container.
 
-- 
cgit v1.2.3


From 2a891aaf38c35534d496b660d71369dd77ff381b Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 5 Oct 2015 18:09:34 -0300
Subject: [bug] handle MissingDesignDocError after get_sync_info

MissingDesignDocError raised on get_sync_info due to a missing design
document will be handled by the server during sync.
Ensure is now False by default, and thus database creation can deliver
an empty one that will be ensured during sync, following the ensure
parameter.
---
 client/changes/bug_missing_design_doc_handler | 1 +
 client/src/leap/soledad/client/sync.py        | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 client/changes/bug_missing_design_doc_handler

diff --git a/client/changes/bug_missing_design_doc_handler b/client/changes/bug_missing_design_doc_handler
new file mode 100644
index 00000000..72e42b85
--- /dev/null
+++ b/client/changes/bug_missing_design_doc_handler
@@ -0,0 +1 @@
+o Handle missing design doc at GET (get_sync_info). Soledad server can handle this during sync.
diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py
index 225d3e2d..2276db2a 100644
--- a/client/src/leap/soledad/client/sync.py
+++ b/client/src/leap/soledad/client/sync.py
@@ -22,6 +22,7 @@ import logging
 from twisted.internet import defer
 
 from u1db import errors
+from leap.soledad.common.errors import MissingDesignDocError
 from u1db.sync import Synchronizer
 
 
@@ -73,8 +74,9 @@ class SoledadSynchronizer(Synchronizer):
             (self.target_replica_uid, target_gen, target_trans_id,
              target_my_gen, target_my_trans_id) = yield \
                 sync_target.get_sync_info(self.source._replica_uid)
-        except errors.DatabaseDoesNotExist:
+        except (errors.DatabaseDoesNotExist, MissingDesignDocError) as e:
             logger.debug("Database isn't ready on server. Will be created.")
+            logger.debug("Reason: %s", e.__class__)
             self.target_replica_uid = None
             target_gen, target_trans_id = 0, ''
             target_my_gen, target_my_trans_id = 0, ''
-- 
cgit v1.2.3


From 1b07d2b54208d6b19135e771162f7e39e02c1f97 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 14 Oct 2015 11:50:48 -0300
Subject: [bug] reduce overall sync session caching to 120s

It was 3600s, but closing connections seems to yet depend on garbage
collection and now causes server to leak file handlers. 120s should be
enough to a sync session finish. Also, lowering this value will only
make very long syncs use more of couch every 2 minutes, while raising
this value will keep memory busy for useless time.
---
 server/src/leap/soledad/server/sync.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index 619be565..b3fb2b84 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -185,7 +185,7 @@ class SyncResource(http_app.SyncResource):
         :type ensure: bool
         """
         # create or open the database
-        cache = get_cache_for('db-' + sync_id + self.dbname)
+        cache = get_cache_for('db-' + sync_id + self.dbname, expire=120)
         if ensure:
             db, self.replica_uid = self.state.ensure_database(self.dbname)
         elif cache and 'instance' in cache:
-- 
cgit v1.2.3


From 0f8364b6a38793f9b0f8596c1b98a590131fbb41 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 19 Oct 2015 16:27:43 -0300
Subject: [bug] remove instance caching

couchdb library relies on garbage collector to close remaining
connections. Somehow, caching the instance is avoiding gc to call
__del__ on underlying couchdb sessions.
---
 server/src/leap/soledad/server/sync.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index b3fb2b84..92b29102 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -188,12 +188,9 @@ class SyncResource(http_app.SyncResource):
         cache = get_cache_for('db-' + sync_id + self.dbname, expire=120)
         if ensure:
             db, self.replica_uid = self.state.ensure_database(self.dbname)
-        elif cache and 'instance' in cache:
-            db = cache['instance']
         else:
             db = self.state.open_database(self.dbname)
         db.init_caching(cache)
-        cache['instance'] = db
         # validate the information the client has about server replica
         db.validate_gen_and_trans_id(
             last_known_generation, last_known_trans_id)
-- 
cgit v1.2.3


From f8d38125098829fe50199725545365d6d2a889a6 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Mon, 26 Oct 2015 18:50:20 -0300
Subject: [feat] read security doc from configuration

LEAP Platform needs to granularly allow access on user database for
other services, like mx. This is now possible by editing
soledad-server.conf file. A new section 'database-security' was added
and it is parsed during 'create-user-db' to be set on security design
document, present on every per-user database.
---
 common/src/leap/soledad/common/couch.py            | 23 ++++++++++----
 .../leap/soledad/common/tests/fixture_soledad.conf | 10 ++++++
 common/src/leap/soledad/common/tests/test_couch.py | 24 +++++++++++++--
 .../src/leap/soledad/common/tests/test_server.py   | 36 ++++++++++++++++++++++
 server/pkg/create-user-db                          |  7 +++--
 server/src/leap/soledad/server/__init__.py         | 34 ++++++++++++++------
 6 files changed, 115 insertions(+), 19 deletions(-)
 create mode 100644 common/src/leap/soledad/common/tests/fixture_soledad.conf

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 3dee1473..a801a044 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -372,7 +372,8 @@ class CouchDatabase(CommonBackend):
     """
 
     @classmethod
-    def open_database(cls, url, create, replica_uid=None, ensure_ddocs=False):
+    def open_database(cls, url, create, replica_uid=None, ensure_ddocs=False,
+                      database_security=None):
         """
         Open a U1DB database using CouchDB as backend.
 
@@ -402,9 +403,11 @@ class CouchDatabase(CommonBackend):
                     raise DatabaseDoesNotExist()
                 server.create(dbname)
         return cls(
-            url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs)
+            url, dbname, replica_uid=replica_uid,
+            ensure_ddocs=ensure_ddocs, database_security=database_security)
 
-    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=False):
+    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=False,
+                 database_security=None):
         """
         Create a new Couch data container.
 
@@ -435,7 +438,7 @@ class CouchDatabase(CommonBackend):
             self._set_replica_uid(replica_uid)
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
-            self.ensure_security_ddoc()
+            self.ensure_security_ddoc(database_security)
         self._cache = None
 
     @property
@@ -469,7 +472,7 @@ class CouchDatabase(CommonBackend):
                         getattr(ddocs, ddoc_name)))
                 self._database.save(ddoc)
 
-    def ensure_security_ddoc(self):
+    def ensure_security_ddoc(self, security_config=None):
         """
         Make sure that only soledad user is able to access this database as
         an unprivileged member, meaning that administration access will
@@ -478,10 +481,18 @@ class CouchDatabase(CommonBackend):
         to the unprivileged CouchDB user set on the server process.
         This is achieved by creating a _security design document, see:
         http://docs.couchdb.org/en/latest/api/database/security.html
+
+        :param database_security: security configuration parsed from conf file
+        :type cache: dict
         """
+        security_config = security_config or {}
         security = self._database.resource.get_json('_security')[2]
-        security['members'] = {'names': ['soledad'], 'roles': []}
+        security['members'] = {'names': [], 'roles': []}
+        security['members']['names'] = security_config.get('members', ['soledad'])
+        security['members']['roles'] = security_config.get('members_roles', [])
         security['admins'] = {'names': [], 'roles': []}
+        security['admins']['names'] = security_config.get('admins', [])
+        security['admins']['roles'] = security_config.get('admins_roles', [])
         self._database.resource.put_json('_security', body=security)
 
     def get_sync_target(self):
diff --git a/common/src/leap/soledad/common/tests/fixture_soledad.conf b/common/src/leap/soledad/common/tests/fixture_soledad.conf
new file mode 100644
index 00000000..c0ffacf6
--- /dev/null
+++ b/common/src/leap/soledad/common/tests/fixture_soledad.conf
@@ -0,0 +1,10 @@
+[soledad-server]
+couch_url   = http://soledad:passwd@localhost:5984
+create_cmd  = sudo -u soledad-admin /usr/bin/create-user-db
+admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc
+
+[database-security]
+members       = user1, user2
+members_roles = role1, role2
+admins        = user3, user4
+admins_roles  = role3, role3
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index b4797f5e..668a7f64 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -534,9 +534,11 @@ class CouchDatabaseSyncTargetTests(
 
 class IndexedCouchDatabase(couch.CouchDatabase):
 
-    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True):
+    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True,
+                 database_security=None):
         old_class.__init__(self, url, dbname, replica_uid=replica_uid,
-                           ensure_ddocs=ensure_ddocs)
+                           ensure_ddocs=ensure_ddocs,
+                           database_security=database_security)
         self._indexes = {}
 
     def _put_doc(self, old_doc, doc):
@@ -1515,6 +1517,24 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         self.assertIn('members', security_ddoc)
         self.assertIn('soledad', security_ddoc['members']['names'])
 
+    def test_ensure_security_from_configuration(self):
+        """
+        Given a configuration, follow it to create the security document
+        """
+        self.create_db(ensure=False)
+        configuration = {'members': ['user1', 'user2'],
+                         'members_roles': ['role1', 'role2'],
+                         'admins': ['admin'],
+                         'admins_roles': ['administrators']
+                         }
+        self.db.ensure_security_ddoc(configuration)
+
+        security_ddoc = self.db._database.resource.get_json('_security')[2]
+        self.assertEquals(configuration['admins'], security_ddoc['admins']['names'])
+        self.assertEquals(configuration['admins_roles'], security_ddoc['admins']['roles'])
+        self.assertEquals(configuration['members'], security_ddoc['members']['names'])
+        self.assertEquals(configuration['members_roles'], security_ddoc['members']['roles'])
+
 
 class DatabaseNameValidationTest(unittest.TestCase):
 
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index 19d2907d..14e17e67 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -22,10 +22,12 @@ import tempfile
 import mock
 import time
 import binascii
+import pkg_resources
 from uuid import uuid4
 
 from urlparse import urljoin
 from twisted.internet import defer
+from twisted.trial import unittest
 
 from leap.soledad.common.couch import (
     CouchServerState,
@@ -43,6 +45,8 @@ from leap.soledad.common.tests.util import (
 from leap.soledad.common import crypto
 from leap.soledad.client import Soledad
 from leap.soledad.server import LockResource
+from leap.soledad.server import load_configuration
+from leap.soledad.server import CONFIG_DEFAULTS
 from leap.soledad.server.auth import URLToAuthorization
 
 
@@ -587,3 +591,35 @@ class LockResourceTestCase(
         self.assertIsNotNone(lr._shared_db.get_doc('lock-' + lock_uuid))
         responder.send_response_json.assert_called_with(
             401, error='unlock unauthorized')
+
+
+class ConfigurationParsingTest(unittest.TestCase):
+
+    def setUp(self):
+        self.maxDiff = None
+
+    def test_use_defaults_on_failure(self):
+        config = load_configuration('this file will never exist')
+        expected = CONFIG_DEFAULTS
+        self.assertEquals(expected, config)
+
+    def test_security_values_configuration(self):
+        # given
+        config_path = pkg_resources.resource_filename('leap.soledad.common.tests',
+                                                      'fixture_soledad.conf')
+        # when
+        config = load_configuration(config_path)
+
+        # then
+        expected = {'soledad-server': {
+                    'couch_url': 'http://soledad:passwd@localhost:5984',
+                    'create_cmd': 'sudo -u soledad-admin /usr/bin/create-user-db',
+                    'admin_netrc': '/etc/couchdb/couchdb-soledad-admin.netrc',
+                    },
+                    'database-security': {
+                    'members': ['user1', 'user2'],
+                    'members_roles': ['role1', 'role2'],
+                    'admins': ['user3', 'user4'],
+                    'admins_roles': ['role3', 'role3'],
+        }}
+        self.assertDictEqual(expected, config)
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index 7eafc945..28d1cbd0 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -31,7 +31,8 @@ This is meant to be used by Soledad Server.
 parser = argparse.ArgumentParser(description=description)
 parser.add_argument('dbname', metavar='user-d34db33f', type=str,
                     help='database name on the format user-{uuid4}')
-NETRC_PATH = load_configuration('/etc/soledad/soledad-server.conf')['admin_netrc']
+CONF = load_configuration('/etc/soledad/soledad-server.conf')
+NETRC_PATH = CONF['soledad-server']['admin_netrc']
 
 
 def url_for_db(dbname):
@@ -54,7 +55,9 @@ if __name__ == '__main__':
         print ("Invalid name! %s" % args.dbname)
         sys.exit(1)
     url = url_for_db(args.dbname)
+    db_security = CONF['database-security']
     db = CouchDatabase.open_database(url=url, create=True,
-                                     replica_uid=None, ensure_ddocs=True)
+                                     replica_uid=None, ensure_ddocs=True,
+                                     database_security=db_security)
     print ('success! Created %s, replica_uid: %s' %
            (db._dbname, db.replica_uid))
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index f64d07bf..4d03c82a 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -272,6 +272,20 @@ http_app.HTTPInvocationByMethodWithBody = HTTPInvocationByMethodWithBody
 # ----------------------------------------------------------------------------
 # Auxiliary functions
 # ----------------------------------------------------------------------------
+CONFIG_DEFAULTS = {
+    'soledad-server': {
+        'couch_url': 'http://localhost:5984',
+        'create_cmd': None,
+        'admin_netrc': '/etc/couchdb/couchdb-admin.netrc',
+    },
+    'database-security': {
+        'members': ['soledad'],
+        'members_roles': [],
+        'admins': [],
+        'admins_roles': []
+    }
+}
+
 
 def load_configuration(file_path):
     """
@@ -283,17 +297,18 @@ def load_configuration(file_path):
     @return: A dictionary with the configuration.
     @rtype: dict
     """
-    defaults = {
-        'couch_url': 'http://localhost:5984',
-        'create_cmd': None,
-        'admin_netrc': '/etc/couchdb/couchdb-admin.netrc',
-    }
+    defaults = dict(CONFIG_DEFAULTS)
     config = configparser.ConfigParser()
     config.read(file_path)
-    if 'soledad-server' in config:
-        for key in defaults:
-            if key in config['soledad-server']:
-                defaults[key] = config['soledad-server'][key]
+    for section in defaults.keys():
+        if section in config:
+            for key in defaults[section]:
+                if key in config[section]:
+                    defaults[section][key] = config[section][key]
+    for key, value in defaults['database-security'].iteritems():
+        if type(value) is not unicode: continue
+        defaults['database-security'][key] = \
+                [item.strip() for item in value.split(',')]
     # TODO: implement basic parsing/sanitization of options comming from
     # config file.
     return defaults
@@ -305,6 +320,7 @@ def load_configuration(file_path):
 
 def application(environ, start_response):
     conf = load_configuration('/etc/soledad/soledad-server.conf')
+    conf = conf['soledad-server']
     state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'])
     # WSGI application that may be used by `twistd -web`
     application = GzipMiddleware(
-- 
cgit v1.2.3


From 7c50a100a46c69f759dc22165cec9b8098cac3a4 Mon Sep 17 00:00:00 2001
From: Folker Bernitt <fbernitt@thoughtworks.com>
Date: Tue, 27 Oct 2015 10:20:56 +0100
Subject: [style] fix pep8 warnigs

---
 common/src/leap/soledad/common/couch.py             |  3 ++-
 common/src/leap/soledad/common/tests/test_couch.py  | 12 ++++++++----
 common/src/leap/soledad/common/tests/test_server.py | 15 +++++++++------
 server/src/leap/soledad/server/__init__.py          |  3 ++-
 4 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index a801a044..c2809a47 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -488,7 +488,8 @@ class CouchDatabase(CommonBackend):
         security_config = security_config or {}
         security = self._database.resource.get_json('_security')[2]
         security['members'] = {'names': [], 'roles': []}
-        security['members']['names'] = security_config.get('members', ['soledad'])
+        security['members']['names'] = security_config.get('members',
+                                                           ['soledad'])
         security['members']['roles'] = security_config.get('members_roles', [])
         security['admins'] = {'names': [], 'roles': []}
         security['admins']['names'] = security_config.get('admins', [])
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 668a7f64..86cc0881 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -1530,10 +1530,14 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         self.db.ensure_security_ddoc(configuration)
 
         security_ddoc = self.db._database.resource.get_json('_security')[2]
-        self.assertEquals(configuration['admins'], security_ddoc['admins']['names'])
-        self.assertEquals(configuration['admins_roles'], security_ddoc['admins']['roles'])
-        self.assertEquals(configuration['members'], security_ddoc['members']['names'])
-        self.assertEquals(configuration['members_roles'], security_ddoc['members']['roles'])
+        self.assertEquals(configuration['admins'],
+                          security_ddoc['admins']['names'])
+        self.assertEquals(configuration['admins_roles'],
+                          security_ddoc['admins']['roles'])
+        self.assertEquals(configuration['members'],
+                          security_ddoc['members']['names'])
+        self.assertEquals(configuration['members_roles'],
+                          security_ddoc['members']['roles'])
 
 
 class DatabaseNameValidationTest(unittest.TestCase):
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index 14e17e67..7c006121 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -22,7 +22,7 @@ import tempfile
 import mock
 import time
 import binascii
-import pkg_resources
+from pkg_resources import resource_filename
 from uuid import uuid4
 
 from urlparse import urljoin
@@ -605,16 +605,19 @@ class ConfigurationParsingTest(unittest.TestCase):
 
     def test_security_values_configuration(self):
         # given
-        config_path = pkg_resources.resource_filename('leap.soledad.common.tests',
-                                                      'fixture_soledad.conf')
+        config_path = resource_filename('leap.soledad.common.tests',
+                                        'fixture_soledad.conf')
         # when
         config = load_configuration(config_path)
 
         # then
         expected = {'soledad-server': {
-                    'couch_url': 'http://soledad:passwd@localhost:5984',
-                    'create_cmd': 'sudo -u soledad-admin /usr/bin/create-user-db',
-                    'admin_netrc': '/etc/couchdb/couchdb-soledad-admin.netrc',
+                    'couch_url':
+                        'http://soledad:passwd@localhost:5984',
+                    'create_cmd':
+                        'sudo -u soledad-admin /usr/bin/create-user-db',
+                    'admin_netrc':
+                        '/etc/couchdb/couchdb-soledad-admin.netrc',
                     },
                     'database-security': {
                     'members': ['user1', 'user2'],
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 4d03c82a..fe67e45f 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -306,7 +306,8 @@ def load_configuration(file_path):
                 if key in config[section]:
                     defaults[section][key] = config[section][key]
     for key, value in defaults['database-security'].iteritems():
-        if type(value) is not unicode: continue
+        if type(value) is not unicode:
+            continue
         defaults['database-security'][key] = \
                 [item.strip() for item in value.split(',')]
     # TODO: implement basic parsing/sanitization of options comming from
-- 
cgit v1.2.3


From bc8e8bea61b01e0b93593e24a816574f5a19ac61 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 27 Oct 2015 13:30:07 -0300
Subject: [feat] adds --migrate-all to create-user-db script

This parameter applies the ensure_database to all existing databases,
making all of them use the latest design documents.
This can be used to migrate security documents, update handlers or any
other kind of design document.
---
 common/src/leap/soledad/common/couch.py | 17 ++++++++++++++++
 server/changes/feat_configurable_ensure |  5 +++++
 server/pkg/create-user-db               | 35 +++++++++++++++++++++++++++------
 3 files changed, 51 insertions(+), 6 deletions(-)
 create mode 100644 server/changes/feat_configurable_ensure

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index c2809a47..ccedef15 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -70,6 +70,23 @@ logger = logging.getLogger(__name__)
 COUCH_TIMEOUT = 120  # timeout for transfers between Soledad server and Couch
 
 
+def list_users_dbs(couch_url):
+    """
+    Retrieves a list with all databases that starts with 'user-' on CouchDB.
+    Those databases belongs to users. So, the list will contain all the
+    database names in the form of 'user-{uuid4}'.
+
+    :param couch_url: The couch url with needed credentials
+    :type couch_url: str
+
+    :return: The list of all database names from users.
+    :rtype: [str]
+    """
+    with couch_server(couch_url) as server:
+        users = [dbname for dbname in server if dbname.startswith('user-')]
+    return users
+
+
 class InvalidURLError(Exception):
 
     """
diff --git a/server/changes/feat_configurable_ensure b/server/changes/feat_configurable_ensure
new file mode 100644
index 00000000..34a20c86
--- /dev/null
+++ b/server/changes/feat_configurable_ensure
@@ -0,0 +1,5 @@
+o 'create-user-db' script now can be configured from soledad-server.conf
+  when generating the user's security document.
+o Migrating a user's database to newest design documents is now possible by
+  using a parameter '--migrate-all' on 'create-user-db' script.
+ 
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index 28d1cbd0..ae5d15dc 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -21,6 +21,7 @@ import netrc
 import argparse
 from leap.soledad.common.couch import CouchDatabase
 from leap.soledad.common.couch import is_db_name_valid
+from leap.soledad.common.couch import list_users_dbs
 from leap.soledad.server import load_configuration
 
 
@@ -30,7 +31,10 @@ This is meant to be used by Soledad Server.
 """
 parser = argparse.ArgumentParser(description=description)
 parser.add_argument('dbname', metavar='user-d34db33f', type=str,
+                    default='', nargs='?',
                     help='database name on the format user-{uuid4}')
+parser.add_argument('--migrate-all', action='store_true',
+                    help="recreate all design docs for all existing account")
 CONF = load_configuration('/etc/soledad/soledad-server.conf')
 NETRC_PATH = CONF['soledad-server']['admin_netrc']
 
@@ -49,15 +53,34 @@ def url_for_db(dbname):
     return url
 
 
-if __name__ == '__main__':
-    args = parser.parse_args()
-    if not is_db_name_valid(args.dbname):
-        print ("Invalid name! %s" % args.dbname)
+def ensure_database(dbname):
+    """
+    This method will ensure that a database named `dbname` will exist
+    or created if it doesn't. Calling it twice will ensure that design
+    documents are present and updated.
+    The database name has to match this criteria to be considered valid:
+    user-[a-f0-9]+
+
+    :param dbname: name of the user database
+    :type dbname: str
+    """
+    if not is_db_name_valid(dbname):
+        print ("Invalid name! %s" % dbname)
         sys.exit(1)
-    url = url_for_db(args.dbname)
+    url = url_for_db(dbname)
     db_security = CONF['database-security']
     db = CouchDatabase.open_database(url=url, create=True,
                                      replica_uid=None, ensure_ddocs=True,
                                      database_security=db_security)
-    print ('success! Created %s, replica_uid: %s' %
+    print ('success! Ensured that database %s exists, with replica_uid: %s' %
            (db._dbname, db.replica_uid))
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    if args.migrate_all:
+        couch_url = url_for_db('')
+        for dbname in list_users_dbs(couch_url):
+            ensure_database(dbname)
+    else:
+        ensure_database(args.dbname)
-- 
cgit v1.2.3


From 3b869fb7ffedc88c738e0a17347b9506d242cabe Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 27 Oct 2015 13:58:56 -0300
Subject: [feat] remove SSL tsafe monkeypatch

This was added for a Twisted 12 bug that should be gone by now.
---
 server/changes/feat_configurable_ensure    |  2 +-
 server/src/leap/soledad/server/__init__.py | 13 -------------
 2 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/server/changes/feat_configurable_ensure b/server/changes/feat_configurable_ensure
index 34a20c86..8abd2ac9 100644
--- a/server/changes/feat_configurable_ensure
+++ b/server/changes/feat_configurable_ensure
@@ -2,4 +2,4 @@ o 'create-user-db' script now can be configured from soledad-server.conf
   when generating the user's security document.
 o Migrating a user's database to newest design documents is now possible by
   using a parameter '--migrate-all' on 'create-user-db' script.
- 
+o Remove tsafe monkeypatch from SSL lib, as it was needed for Twisted <12
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index fe67e45f..618ccb2b 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -94,12 +94,6 @@ from u1db.remote import http_app, utils
 
 from ._version import get_versions
 
-# Keep OpenSSL's tsafe before importing Twisted submodules so we can put
-# it back if Twisted==12.0.0 messes with it.
-from OpenSSL import tsafe
-
-from twisted import version
-
 from leap.soledad.server.auth import SoledadTokenAuthMiddleware
 from leap.soledad.server.gzip_middleware import GzipMiddleware
 from leap.soledad.server.lock_resource import LockResource
@@ -112,13 +106,6 @@ from leap.soledad.server.sync import (
 from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common.couch import CouchServerState
 
-old_tsafe = tsafe
-
-if version.base() == "12.0.0":
-    # Put OpenSSL's tsafe back into place. This can probably be removed if we
-    # come to use Twisted>=12.3.0.
-    sys.modules['OpenSSL.tsafe'] = old_tsafe
-
 # ----------------------------------------------------------------------------
 # Soledad WSGI application
 # ----------------------------------------------------------------------------
-- 
cgit v1.2.3


From 36f476c53f6f3532652d7428e306dcef1c37f879 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Fri, 30 Oct 2015 16:26:07 +0100
Subject: [feat] bump twisted dependency to 12.3.0

Needed after the SSL monkeypatch removal at 3b869fb7f
---
 server/pkg/requirements.pip | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip
index 58834d0e..f9cce08e 100644
--- a/server/pkg/requirements.pip
+++ b/server/pkg/requirements.pip
@@ -2,7 +2,7 @@ configparser
 u1db
 routes
 PyOpenSSL
-twisted
+twisted>=12.3.0
 #pinned for wheezy compatibility
 Beaker==1.6.3 #wheezy
 couchdb==0.8 #wheezy
-- 
cgit v1.2.3


From bd316723244908010e81a6b745313df254b3004a Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 30 Sep 2015 17:12:44 -0300
Subject: [refactor] separate errors and model from couch

CouchDocument is a subclass from SoledadDocument which holds conflicts,
making the server side aware of them. There is a module called document
on soledad.common that holds SoledadDocument. This commit moves
CouchDocument to its proper place.
Error raising logic and exceptions declarations are also moved into error
module inside common.
---
 common/src/leap/soledad/common/couch.py    | 217 ++++-------------------------
 common/src/leap/soledad/common/document.py |  82 +++++++++++
 common/src/leap/soledad/common/errors.py   |  71 ++++++++++
 3 files changed, 182 insertions(+), 188 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index ccedef15..9bdafc45 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -59,9 +59,12 @@ from u1db.remote import http_app
 from u1db.remote.server_state import ServerState
 
 
-from leap.soledad.common import ddocs, errors
+from leap.soledad.common import ddocs
+from leap.soledad.common.errors import raise_server_error
+from leap.soledad.common.errors import raise_missing_design_doc_error
+from leap.soledad.common.errors import InvalidURLError
 from leap.soledad.common.command import exec_validated_cmd
-from leap.soledad.common.document import SoledadDocument
+from leap.soledad.common.document import CouchDocument
 
 
 logger = logging.getLogger(__name__)
@@ -86,193 +89,10 @@ def list_users_dbs(couch_url):
         users = [dbname for dbname in server if dbname.startswith('user-')]
     return users
 
-
-class InvalidURLError(Exception):
-
-    """
-    Exception raised when Soledad encounters a malformed URL.
-    """
-
-
-class CouchDocument(SoledadDocument):
-
-    """
-    This is the document used for maintaining the Couch backend.
-
-    A CouchDocument can fetch and manipulate conflicts and also holds a
-    reference to the couch document revision. This data is used to ensure an
-    atomic and consistent update of the database.
-    """
-
-    def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False):
-        """
-        Container for handling a document that is stored in couch backend.
-
-        :param doc_id: The unique document identifier.
-        :type doc_id: str
-        :param rev: The revision identifier of the document.
-        :type rev: str
-        :param json: The JSON string for this document.
-        :type json: str
-        :param has_conflicts: Boolean indicating if this document has conflicts
-        :type has_conflicts: bool
-        """
-        SoledadDocument.__init__(self, doc_id, rev, json, has_conflicts)
-        self.couch_rev = None
-        self.transactions = None
-        self._conflicts = None
-
-    def get_conflicts(self):
-        """
-        Get the conflicted versions of the document.
-
-        :return: The conflicted versions of the document.
-        :rtype: [CouchDocument]
-        """
-        return self._conflicts or []
-
-    def set_conflicts(self, conflicts):
-        """
-        Set the conflicted versions of the document.
-
-        :param conflicts: The conflicted versions of the document.
-        :type conflicts: list
-        """
-        self._conflicts = conflicts
-        self.has_conflicts = len(self._conflicts) > 0
-
-    def add_conflict(self, doc):
-        """
-        Add a conflict to this document.
-
-        :param doc: The conflicted version to be added.
-        :type doc: CouchDocument
-        """
-        if self._conflicts is None:
-            raise Exception("Fetch conflicts first!")
-        self._conflicts.append(doc)
-        self.has_conflicts = len(self._conflicts) > 0
-
-    def delete_conflicts(self, conflict_revs):
-        """
-        Delete conflicted versions of this document.
-
-        :param conflict_revs: The conflicted revisions to be deleted.
-        :type conflict_revs: [str]
-        """
-        if self._conflicts is None:
-            raise Exception("Fetch conflicts first!")
-        self._conflicts = filter(
-            lambda doc: doc.rev not in conflict_revs,
-            self._conflicts)
-        self.has_conflicts = len(self._conflicts) > 0
-
-    def update(self, new_doc):
-        # update info
-        self.rev = new_doc.rev
-        if new_doc.is_tombstone():
-            self.is_tombstone()
-        else:
-            self.content = new_doc.content
-        self.has_conflicts = new_doc.has_conflicts
-
-    def prune_conflicts(self, doc_vcr, autoresolved_increment):
-        """
-        Prune conflicts that are older then the current document's revision, or
-        whose content match to the current document's content.
-        Originally in u1db.CommonBackend
-
-        :param doc: The document to have conflicts pruned.
-        :type doc: CouchDocument
-        :param doc_vcr: A vector clock representing the current document's
-                        revision.
-        :type doc_vcr: u1db.vectorclock.VectorClock
-        """
-        if self.has_conflicts:
-            autoresolved = False
-            c_revs_to_prune = []
-            for c_doc in self._conflicts:
-                c_vcr = vectorclock.VectorClockRev(c_doc.rev)
-                if doc_vcr.is_newer(c_vcr):
-                    c_revs_to_prune.append(c_doc.rev)
-                elif self.same_content_as(c_doc):
-                    c_revs_to_prune.append(c_doc.rev)
-                    doc_vcr.maximize(c_vcr)
-                    autoresolved = True
-            if autoresolved:
-                doc_vcr.increment(autoresolved_increment)
-                self.rev = doc_vcr.as_str()
-            self.delete_conflicts(c_revs_to_prune)
-
-
 # monkey-patch the u1db http app to use CouchDocument
 http_app.Document = CouchDocument
 
 
-def raise_missing_design_doc_error(exc, ddoc_path):
-    """
-    Raise an appropriate exception when catching a ResourceNotFound when
-    accessing a design document.
-
-    :param exc: The exception cought.
-    :type exc: ResourceNotFound
-    :param ddoc_path: A list representing the requested path.
-    :type ddoc_path: list
-
-    :raise MissingDesignDocError: Raised when tried to access a missing design
-                                  document.
-    :raise MissingDesignDocListFunctionError: Raised when trying to access a
-                                              missing list function on a
-                                              design document.
-    :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                           missing named view on a design
-                                           document.
-    :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                         deleted design document.
-    :raise MissingDesignDocUnknownError: Raised when failed to access a design
-                                         document for an yet unknown reason.
-    """
-    path = "".join(ddoc_path)
-    if exc.message[1] == 'missing':
-        raise errors.MissingDesignDocError(path)
-    elif exc.message[1] == 'missing function' or \
-            exc.message[1].startswith('missing lists function'):
-        raise errors.MissingDesignDocListFunctionError(path)
-    elif exc.message[1] == 'missing_named_view':
-        raise errors.MissingDesignDocNamedViewError(path)
-    elif exc.message[1] == 'deleted':
-        raise errors.MissingDesignDocDeletedError(path)
-    # other errors are unknown for now
-    raise errors.DesignDocUnknownError("%s: %s" % (path, str(exc.message)))
-
-
-def raise_server_error(exc, ddoc_path):
-    """
-    Raise an appropriate exception when catching a ServerError when
-    accessing a design document.
-
-    :param exc: The exception cought.
-    :type exc: ResourceNotFound
-    :param ddoc_path: A list representing the requested path.
-    :type ddoc_path: list
-
-    :raise MissingDesignDocListFunctionError: Raised when trying to access a
-                                              missing list function on a
-                                              design document.
-    :raise MissingDesignDocUnknownError: Raised when failed to access a design
-                                         document for an yet unknown reason.
-    """
-    path = "".join(ddoc_path)
-    msg = exc.message[1][0]
-    if msg == 'unnamed_error':
-        raise errors.MissingDesignDocListFunctionError(path)
-    elif msg == 'TypeError':
-        if 'point is undefined' in exc.message[1][1]:
-            raise errors.MissingDesignDocListFunctionError
-    # other errors are unknown for now
-    raise errors.DesignDocUnknownError(path)
-
-
 class MultipartWriter(object):
 
     """
@@ -1389,10 +1209,31 @@ class CouchDatabase(CommonBackend):
 
     def _prune_conflicts(self, doc, doc_vcr):
         """
-        Overrides original method, but it is implemented elsewhere for
-        simplicity.
+        Prune conflicts that are older then the current document's revision, or
+        whose content match to the current document's content.
+        Originally in u1db.CommonBackend
+
+        :param doc: The document to have conflicts pruned.
+        :type doc: CouchDocument
+        :param doc_vcr: A vector clock representing the current document's
+                        revision.
+        :type doc_vcr: u1db.vectorclock.VectorClock
         """
-        doc.prune_conflicts(doc_vcr, self._replica_uid)
+        if doc.has_conflicts:
+            autoresolved = False
+            c_revs_to_prune = []
+            for c_doc in doc._conflicts:
+                c_vcr = vectorclock.VectorClockRev(c_doc.rev)
+                if doc_vcr.is_newer(c_vcr):
+                    c_revs_to_prune.append(c_doc.rev)
+                elif doc.same_content_as(c_doc):
+                    c_revs_to_prune.append(c_doc.rev)
+                    doc_vcr.maximize(c_vcr)
+                    autoresolved = True
+            if autoresolved:
+                doc_vcr.increment(self._replica_uid)
+                doc.rev = doc_vcr.as_str()
+            doc.delete_conflicts(c_revs_to_prune)
 
     def _new_resource(self, *path):
         """
diff --git a/common/src/leap/soledad/common/document.py b/common/src/leap/soledad/common/document.py
index 919ade12..05458906 100644
--- a/common/src/leap/soledad/common/document.py
+++ b/common/src/leap/soledad/common/document.py
@@ -108,3 +108,85 @@ class SoledadDocument(Document):
         _get_rev,
         _set_rev,
         doc="Wrapper to ensure `doc.rev` is always returned as bytes.")
+
+
+class CouchDocument(SoledadDocument):
+    """
+    This is the document used for maintaining the Couch backend.
+
+    A CouchDocument can fetch and manipulate conflicts and also holds a
+    reference to the couch document revision. This data is used to ensure an
+    atomic and consistent update of the database.
+    """
+
+    def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False):
+        """
+        Container for handling a document that is stored in couch backend.
+
+        :param doc_id: The unique document identifier.
+        :type doc_id: str
+        :param rev: The revision identifier of the document.
+        :type rev: str
+        :param json: The JSON string for this document.
+        :type json: str
+        :param has_conflicts: Boolean indicating if this document has conflicts
+        :type has_conflicts: bool
+        """
+        SoledadDocument.__init__(self, doc_id, rev, json, has_conflicts)
+        self.couch_rev = None
+        self.transactions = None
+        self._conflicts = None
+
+    def get_conflicts(self):
+        """
+        Get the conflicted versions of the document.
+
+        :return: The conflicted versions of the document.
+        :rtype: [CouchDocument]
+        """
+        return self._conflicts or []
+
+    def set_conflicts(self, conflicts):
+        """
+        Set the conflicted versions of the document.
+
+        :param conflicts: The conflicted versions of the document.
+        :type conflicts: list
+        """
+        self._conflicts = conflicts
+        self.has_conflicts = len(self._conflicts) > 0
+
+    def add_conflict(self, doc):
+        """
+        Add a conflict to this document.
+
+        :param doc: The conflicted version to be added.
+        :type doc: CouchDocument
+        """
+        if self._conflicts is None:
+            raise Exception("Fetch conflicts first!")
+        self._conflicts.append(doc)
+        self.has_conflicts = len(self._conflicts) > 0
+
+    def delete_conflicts(self, conflict_revs):
+        """
+        Delete conflicted versions of this document.
+
+        :param conflict_revs: The conflicted revisions to be deleted.
+        :type conflict_revs: [str]
+        """
+        if self._conflicts is None:
+            raise Exception("Fetch conflicts first!")
+        self._conflicts = filter(
+            lambda doc: doc.rev not in conflict_revs,
+            self._conflicts)
+        self.has_conflicts = len(self._conflicts) > 0
+
+    def update(self, new_doc):
+        # update info
+        self.rev = new_doc.rev
+        if new_doc.is_tombstone():
+            self.is_tombstone()
+        else:
+            self.content = new_doc.content
+        self.has_conflicts = new_doc.has_conflicts
diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py
index 5798770b..2ba05839 100644
--- a/common/src/leap/soledad/common/errors.py
+++ b/common/src/leap/soledad/common/errors.py
@@ -196,3 +196,74 @@ class DesignDocUnknownError(SoledadError):
 # u1db error statuses also have to be updated
 http_errors.ERROR_STATUSES = set(
     http_errors.wire_description_to_status.values())
+
+
+class InvalidURLError(Exception):
+
+    """
+    Exception raised when Soledad encounters a malformed URL.
+    """
+
+
+def raise_missing_design_doc_error(exc, ddoc_path):
+    """
+    Raise an appropriate exception when catching a ResourceNotFound when
+    accessing a design document.
+
+    :param exc: The exception cought.
+    :type exc: ResourceNotFound
+    :param ddoc_path: A list representing the requested path.
+    :type ddoc_path: list
+
+    :raise MissingDesignDocError: Raised when tried to access a missing design
+                                  document.
+    :raise MissingDesignDocListFunctionError: Raised when trying to access a
+                                              missing list function on a
+                                              design document.
+    :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                           missing named view on a design
+                                           document.
+    :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                         deleted design document.
+    :raise MissingDesignDocUnknownError: Raised when failed to access a design
+                                         document for an yet unknown reason.
+    """
+    path = "".join(ddoc_path)
+    if exc.message[1] == 'missing':
+        raise MissingDesignDocError(path)
+    elif exc.message[1] == 'missing function' or \
+            exc.message[1].startswith('missing lists function'):
+        raise MissingDesignDocListFunctionError(path)
+    elif exc.message[1] == 'missing_named_view':
+        raise MissingDesignDocNamedViewError(path)
+    elif exc.message[1] == 'deleted':
+        raise MissingDesignDocDeletedError(path)
+    # other errors are unknown for now
+    raise DesignDocUnknownError("%s: %s" % (path, str(exc.message)))
+
+
+def raise_server_error(exc, ddoc_path):
+    """
+    Raise an appropriate exception when catching a ServerError when
+    accessing a design document.
+
+    :param exc: The exception cought.
+    :type exc: ResourceNotFound
+    :param ddoc_path: A list representing the requested path.
+    :type ddoc_path: list
+
+    :raise MissingDesignDocListFunctionError: Raised when trying to access a
+                                              missing list function on a
+                                              design document.
+    :raise MissingDesignDocUnknownError: Raised when failed to access a design
+                                         document for an yet unknown reason.
+    """
+    path = "".join(ddoc_path)
+    msg = exc.message[1][0]
+    if msg == 'unnamed_error':
+        raise MissingDesignDocListFunctionError(path)
+    elif msg == 'TypeError':
+        if 'point is undefined' in exc.message[1][1]:
+            raise MissingDesignDocListFunctionError
+    # other errors are unknown for now
+    raise DesignDocUnknownError(path)
-- 
cgit v1.2.3


From 983644df965b391527fce51bda6f6b1f4b29c2ac Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 30 Sep 2015 17:22:08 -0300
Subject: [refactor] remove MultipartWriter monkey patch

This monkey patch was added to separate headers from content and use
them to PUT on couchdb. The original implementation makes it possible by
using two parameters in the constructor.
---
 common/src/leap/soledad/common/couch.py | 94 ++-------------------------------
 1 file changed, 3 insertions(+), 91 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 9bdafc45..4f569559 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -25,7 +25,6 @@ import uuid
 import logging
 import binascii
 import time
-import sys
 
 
 from StringIO import StringIO
@@ -35,6 +34,7 @@ from multiprocessing.pool import ThreadPool
 
 
 from couchdb.client import Server, Database
+from couchdb.multipart import MultipartWriter
 from couchdb.http import (
     ResourceConflict,
     ResourceNotFound,
@@ -93,95 +93,6 @@ def list_users_dbs(couch_url):
 http_app.Document = CouchDocument
 
 
-class MultipartWriter(object):
-
-    """
-    A multipart writer adapted from python-couchdb's one so we can PUT
-    documents using couch's multipart PUT.
-
-    This stripped down version does not allow for nested structures, and
-    contains only the essential things we need to PUT SoledadDocuments to the
-    couch backend.
-    """
-
-    CRLF = '\r\n'
-
-    def __init__(self, fileobj, headers=None, boundary=None):
-        """
-        Initialize the multipart writer.
-        """
-        self.fileobj = fileobj
-        if boundary is None:
-            boundary = self._make_boundary()
-        self._boundary = boundary
-        self._build_headers('related', headers)
-
-    def add(self, mimetype, content, headers={}):
-        """
-        Add a part to the multipart stream.
-        """
-        self.fileobj.write('--')
-        self.fileobj.write(self._boundary)
-        self.fileobj.write(self.CRLF)
-        headers['Content-Type'] = mimetype
-        self._write_headers(headers)
-        if content:
-            # XXX: throw an exception if a boundary appears in the content??
-            self.fileobj.write(content)
-            self.fileobj.write(self.CRLF)
-
-    def close(self):
-        """
-        Close the multipart stream.
-        """
-        self.fileobj.write('--')
-        self.fileobj.write(self._boundary)
-        # be careful not to have anything after '--', otherwise old couch
-        # versions (including bigcouch) will fail.
-        self.fileobj.write('--')
-
-    def _make_boundary(self):
-        """
-        Create a boundary to discern multi parts.
-        """
-        try:
-            from uuid import uuid4
-            return '==' + uuid4().hex + '=='
-        except ImportError:
-            from random import randrange
-            token = randrange(sys.maxint)
-            format = '%%0%dd' % len(repr(sys.maxint - 1))
-            return '===============' + (format % token) + '=='
-
-    def _write_headers(self, headers):
-        """
-        Write a part header in the buffer stream.
-        """
-        if headers:
-            for name in sorted(headers.keys()):
-                value = headers[name]
-                self.fileobj.write(name)
-                self.fileobj.write(': ')
-                self.fileobj.write(value)
-                self.fileobj.write(self.CRLF)
-        self.fileobj.write(self.CRLF)
-
-    def _build_headers(self, subtype, headers):
-        """
-        Build the main headers of the multipart stream.
-
-        This is here so we can send headers separete from content using
-        python-couchdb API.
-        """
-        self.headers = {}
-        self.headers['Content-Type'] = 'multipart/%s; boundary="%s"' % \
-                                       (subtype, self._boundary)
-        if headers:
-            for name in sorted(headers.keys()):
-                value = headers[name]
-                self.headers[name] = value
-
-
 @contextmanager
 def couch_server(url):
     """
@@ -729,7 +640,8 @@ class CouchDatabase(CommonBackend):
             couch_doc['_rev'] = old_doc.couch_rev
         # prepare the multipart PUT
         buf = StringIO()
-        envelope = MultipartWriter(buf)
+        headers = {}
+        envelope = MultipartWriter(buf, headers=headers, subtype='related')
         envelope.add('application/json', json.dumps(couch_doc))
         for part in parts:
             envelope.add('application/octet-stream', part)
-- 
cgit v1.2.3


From b0557f9c1d5e6f153f926ba3cb5876453ef23a10 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Thu, 1 Oct 2015 15:07:25 -0300
Subject: [refactor] separate SoledadBackend from CouchDatabase

CouchDatabase was renamed to SoledadBackend and a new class
CouchDatabase was created to hold all couchdb code. This should make
SoledadBackend less tied to database implementation. A few more
separations are needed to split into modules.
---
 common/src/leap/soledad/common/couch.py            | 1115 ++++++++++++--------
 common/src/leap/soledad/common/errors.py           |    2 +-
 common/src/leap/soledad/common/tests/test_couch.py |   53 +-
 .../tests/test_couch_operations_atomicity.py       |    4 +-
 .../src/leap/soledad/common/tests/test_server.py   |    6 +-
 common/src/leap/soledad/common/tests/test_sync.py  |    2 +-
 .../leap/soledad/common/tests/test_sync_mutex.py   |    2 +-
 .../leap/soledad/common/tests/test_sync_target.py  |    4 +-
 common/src/leap/soledad/common/tests/util.py       |    4 +-
 server/pkg/create-user-db                          |    4 +-
 server/src/leap/soledad/server/sync.py             |    2 +-
 11 files changed, 709 insertions(+), 489 deletions(-)

diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
index 4f569559..d4f67696 100644
--- a/common/src/leap/soledad/common/couch.py
+++ b/common/src/leap/soledad/common/couch.py
@@ -113,7 +113,7 @@ def couch_server(url):
 THREAD_POOL = ThreadPool(20)
 
 
-class CouchDatabase(CommonBackend):
+class SoledadBackend(CommonBackend):
 
     """
     A U1DB implementation that uses CouchDB as its persistence layer.
@@ -135,27 +135,13 @@ class CouchDatabase(CommonBackend):
         :type ensure_ddocs: bool
 
         :return: the database instance
-        :rtype: CouchDatabase
+        :rtype: SoledadBackend
         """
-        # get database from url
-        m = re.match('(^https?://[^/]+)/(.+)$', url)
-        if not m:
-            raise InvalidURLError
-        url = m.group(1)
-        dbname = m.group(2)
-        with couch_server(url) as server:
-            try:
-                server[dbname]
-            except ResourceNotFound:
-                if not create:
-                    raise DatabaseDoesNotExist()
-                server.create(dbname)
+        db = CouchDatabase.open_database(url, create, ensure_ddocs)
         return cls(
-            url, dbname, replica_uid=replica_uid,
-            ensure_ddocs=ensure_ddocs, database_security=database_security)
+            db, replica_uid=replica_uid)
 
-    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=False,
-                 database_security=None):
+    def __init__(self, database, replica_uid=None):
         """
         Create a new Couch data container.
 
@@ -169,25 +155,14 @@ class CouchDatabase(CommonBackend):
         :type ensure_ddocs: bool
         """
         # save params
-        self._url = url
-        self._session = Session(timeout=COUCH_TIMEOUT)
         self._factory = CouchDocument
         self._real_replica_uid = None
         # configure couch
-        self._dbname = dbname
-        self._database = Database(
-            urljoin(self._url, self._dbname),
-            self._session)
-        try:
-            self._database.info()
-        except ResourceNotFound:
-            raise DatabaseDoesNotExist()
+        self._cache = None
+        self._dbname = database._dbname
+        self._database = database
         if replica_uid is not None:
             self._set_replica_uid(replica_uid)
-        if ensure_ddocs:
-            self.ensure_ddocs_on_db()
-            self.ensure_security_ddoc(database_security)
-        self._cache = None
 
     @property
     def cache(self):
@@ -205,45 +180,6 @@ class CouchDatabase(CommonBackend):
         """
         self._cache = cache
 
-    def ensure_ddocs_on_db(self):
-        """
-        Ensure that the design documents used by the backend exist on the
-        couch database.
-        """
-        for ddoc_name in ['docs', 'syncs', 'transactions']:
-            try:
-                self._database.resource('_design',
-                                        ddoc_name, '_info').get_json()
-            except ResourceNotFound:
-                ddoc = json.loads(
-                    binascii.a2b_base64(
-                        getattr(ddocs, ddoc_name)))
-                self._database.save(ddoc)
-
-    def ensure_security_ddoc(self, security_config=None):
-        """
-        Make sure that only soledad user is able to access this database as
-        an unprivileged member, meaning that administration access will
-        be forbidden even inside an user database.
-        The goal is to make sure that only the lowest access level is given
-        to the unprivileged CouchDB user set on the server process.
-        This is achieved by creating a _security design document, see:
-        http://docs.couchdb.org/en/latest/api/database/security.html
-
-        :param database_security: security configuration parsed from conf file
-        :type cache: dict
-        """
-        security_config = security_config or {}
-        security = self._database.resource.get_json('_security')[2]
-        security['members'] = {'names': [], 'roles': []}
-        security['members']['names'] = security_config.get('members',
-                                                           ['soledad'])
-        security['members']['roles'] = security_config.get('members_roles', [])
-        security['admins'] = {'names': [], 'roles': []}
-        security['admins']['names'] = security_config.get('admins', [])
-        security['admins']['roles'] = security_config.get('admins_roles', [])
-        self._database.resource.put_json('_security', body=security)
-
     def get_sync_target(self):
         """
         Return a SyncTarget object, for another u1db to synchronize with.
@@ -257,8 +193,7 @@ class CouchDatabase(CommonBackend):
         """
         Delete a U1DB CouchDB database.
         """
-        with couch_server(self._url) as server:
-            del(server[self._dbname])
+        self._database.delete_database()
 
     def close(self):
         """
@@ -267,10 +202,7 @@ class CouchDatabase(CommonBackend):
         :return: True if db was succesfully closed.
         :rtype: bool
         """
-        self._url = None
-        self._full_commit = None
-        self._session = None
-        self._database = None
+        self._database.close()
         return True
 
     def __del__(self):
@@ -286,18 +218,9 @@ class CouchDatabase(CommonBackend):
         :param replica_uid: The new replica uid.
         :type replica_uid: str
         """
-        try:
-            # set on existent config document
-            doc = self._database['u1db_config']
-            doc['replica_uid'] = replica_uid
-        except ResourceNotFound:
-            # or create the config document
-            doc = {
-                '_id': 'u1db_config',
-                'replica_uid': replica_uid,
-            }
-        self._database.save(doc)
+        self._database.set_replica_uid(replica_uid)
         self._real_replica_uid = replica_uid
+        self.cache['replica_uid'] = self._real_replica_uid
 
     def _get_replica_uid(self):
         """
@@ -307,21 +230,13 @@ class CouchDatabase(CommonBackend):
         :rtype: str
         """
         if self._real_replica_uid is not None:
-            self.cache[self._url] = {'replica_uid': self._real_replica_uid}
-            return self._real_replica_uid
-        if self._url in self.cache:
-            return self.cache[self._url]['replica_uid']
-        try:
-            # grab replica_uid from server
-            doc = self._database['u1db_config']
-            self.cache[self._url] = doc
-            self._real_replica_uid = doc['replica_uid']
-            return self._real_replica_uid
-        except ResourceNotFound:
-            # create a unique replica_uid
-            self._real_replica_uid = uuid.uuid4().hex
-            self._set_replica_uid(self._real_replica_uid)
+            self.cache['replica_uid'] = self._real_replica_uid
             return self._real_replica_uid
+        if 'replica_uid' in self.cache:
+            return self.cache['replica_uid']
+        self._real_replica_uid = self._database.get_replica_uid()
+        self._set_replica_uid(self._real_replica_uid)
+        return self._real_replica_uid
 
     _replica_uid = property(_get_replica_uid, _set_replica_uid)
 
@@ -348,19 +263,7 @@ class CouchDatabase(CommonBackend):
                                              design document for an yet
                                              unknown reason.
         """
-        # query a couch list function
-        if self.replica_uid + '_gen' in self.cache:
-            return self.cache[self.replica_uid + '_gen']['generation']
-        ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json()
-            self.cache[self.replica_uid + '_gen'] = response[2]
-            return response[2]['generation']
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-        except ServerError as e:
-            raise_server_error(e, ddoc_path)
+        return self._get_generation_info()[0]
 
     def _get_generation_info(self):
         """
@@ -385,18 +288,10 @@ class CouchDatabase(CommonBackend):
         """
         if self.replica_uid + '_gen' in self.cache:
             response = self.cache[self.replica_uid + '_gen']
-            return (response['generation'], response['transaction_id'])
-        # query a couch list function
-        ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json()
-            self.cache[self.replica_uid + '_gen'] = response[2]
-            return (response[2]['generation'], response[2]['transaction_id'])
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-        except ServerError as e:
-            raise_server_error(e, ddoc_path)
+            return response
+        cur_gen, newest_trans_id = self._database._get_generation_info()
+        self.cache[self.replica_uid + '_gen'] = (cur_gen, newest_trans_id)
+        return (cur_gen, newest_trans_id)
 
     def _get_trans_id_for_gen(self, generation):
         """
@@ -408,37 +303,8 @@ class CouchDatabase(CommonBackend):
         :return: The transaction id for C{generation}.
         :rtype: str
 
-        :raise InvalidGeneration: Raised when the generation does not exist.
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
         """
-        if generation == 0:
-            return ''
-        # query a couch list function
-        ddoc_path = [
-            '_design', 'transactions', '_list', 'trans_id_for_gen', 'log'
-        ]
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json(gen=generation)
-            if response[2] == {}:
-                raise InvalidGeneration
-            return response[2]['transaction_id']
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-        except ServerError as e:
-            raise_server_error(e, ddoc_path)
+        return self._database._get_trans_id_for_gen(generation)
 
     def _get_transaction_log(self):
         """
@@ -447,30 +313,8 @@ class CouchDatabase(CommonBackend):
         :return: The complete transaction log.
         :rtype: [(str, str)]
 
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
         """
-        # query a couch view
-        ddoc_path = ['_design', 'transactions', '_view', 'log']
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json()
-            return map(
-                lambda row: (row['id'], row['value']),
-                response[2]['rows'])
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
+        return self._database._get_transaction_log()
 
     def _get_doc(self, doc_id, check_for_conflicts=False):
         """
@@ -487,44 +331,7 @@ class CouchDatabase(CommonBackend):
         :return: The document.
         :rtype: CouchDocument
         """
-        # get document with all attachments (u1db content and eventual
-        # conflicts)
-        try:
-            result = \
-                self._database.resource(doc_id).get_json(
-                    attachments=True)[2]
-        except ResourceNotFound:
-            return None
-        return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts)
-
-    def __parse_doc_from_couch(self, result, doc_id,
-                               check_for_conflicts=False):
-        # restrict to u1db documents
-        if 'u1db_rev' not in result:
-            return None
-        doc = self._factory(doc_id, result['u1db_rev'])
-        # set contents or make tombstone
-        if '_attachments' not in result \
-                or 'u1db_content' not in result['_attachments']:
-            doc.make_tombstone()
-        else:
-            doc.content = json.loads(
-                binascii.a2b_base64(
-                    result['_attachments']['u1db_content']['data']))
-        # determine if there are conflicts
-        if check_for_conflicts \
-                and '_attachments' in result \
-                and 'u1db_conflicts' in result['_attachments']:
-            doc.set_conflicts(
-                self._build_conflicts(
-                    doc.doc_id,
-                    json.loads(binascii.a2b_base64(
-                        result['_attachments']['u1db_conflicts']['data']))))
-        # store couch revision
-        doc.couch_rev = result['_rev']
-        # store transactions
-        doc.transactions = result['u1db_transactions']
-        return doc
+        return self._database._get_doc(doc_id, check_for_conflicts)
 
     def get_doc(self, doc_id, include_deleted=False):
         """
@@ -561,11 +368,7 @@ class CouchDatabase(CommonBackend):
             the documents in the database.
         :rtype: (int, [CouchDocument])
         """
-
-        generation = self._get_generation()
-        results = list(self.get_docs(self._database,
-                                     include_deleted=include_deleted))
-        return (generation, results)
+        return self._database.get_all_docs(include_deleted)
 
     def _put_doc(self, old_doc, doc):
         """
@@ -579,86 +382,17 @@ class CouchDatabase(CommonBackend):
         :type old_doc: CouchDocument
         :param doc: The document to be put.
         :type doc: CouchDocument
+        """
+        last_transaction =\
+            self._database.save_document(old_doc, doc,
+                                         self._allocate_transaction_id())
+        if self.replica_uid + '_gen' in self.cache:
+            gen, trans = self.cache[self.replica_uid + '_gen']
+            gen += 1
+            trans = last_transaction
+            self.cache[self.replica_uid + '_gen'] = (gen, trans)
 
-        :raise RevisionConflict: Raised when trying to update a document but
-                                 couch revisions mismatch.
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
-        """
-        attachments = {}  # we save content and conflicts as attachments
-        parts = []  # and we put it using couch's multipart PUT
-        # save content as attachment
-        if doc.is_tombstone() is False:
-            content = doc.get_json()
-            attachments['u1db_content'] = {
-                'follows': True,
-                'content_type': 'application/octet-stream',
-                'length': len(content),
-            }
-            parts.append(content)
-        # save conflicts as attachment
-        if doc.has_conflicts is True:
-            conflicts = json.dumps(
-                map(lambda cdoc: (cdoc.rev, cdoc.content),
-                    doc.get_conflicts()))
-            attachments['u1db_conflicts'] = {
-                'follows': True,
-                'content_type': 'application/octet-stream',
-                'length': len(conflicts),
-            }
-            parts.append(conflicts)
-        # store old transactions, if any
-        transactions = old_doc.transactions[:] if old_doc is not None else []
-        # create a new transaction id and timestamp it so the transaction log
-        # is consistent when querying the database.
-        transactions.append(
-            # here we store milliseconds to keep consistent with javascript
-            # Date.prototype.getTime() which was used before inside a couchdb
-            # update handler.
-            (int(time.time() * 1000),
-             self._allocate_transaction_id()))
-        # build the couch document
-        couch_doc = {
-            '_id': doc.doc_id,
-            'u1db_rev': doc.rev,
-            'u1db_transactions': transactions,
-            '_attachments': attachments,
-        }
-        # if we are updating a doc we have to add the couch doc revision
-        if old_doc is not None:
-            couch_doc['_rev'] = old_doc.couch_rev
-        # prepare the multipart PUT
-        buf = StringIO()
-        headers = {}
-        envelope = MultipartWriter(buf, headers=headers, subtype='related')
-        envelope.add('application/json', json.dumps(couch_doc))
-        for part in parts:
-            envelope.add('application/octet-stream', part)
-        envelope.close()
-        # try to save and fail if there's a revision conflict
-        try:
-            resource = self._new_resource()
-            resource.put_json(
-                doc.doc_id, body=str(buf.getvalue()), headers=envelope.headers)
-        except ResourceConflict:
-            raise RevisionConflict()
-        if self.replica_uid + '_gen' in self.cache:
-            gen_info = self.cache[self.replica_uid + '_gen']
-            gen_info['generation'] += 1
-            gen_info['transaction_id'] = transactions[-1][1]
-
-    def put_doc(self, doc):
+    def put_doc(self, doc):
         """
         Update a document.
 
@@ -711,53 +445,8 @@ class CouchDatabase(CommonBackend):
                  to the last intervening change and sorted by generation (old
                  changes first)
         :rtype: (int, str, [(str, int, str)])
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
         """
-        # query a couch list function
-        ddoc_path = [
-            '_design', 'transactions', '_list', 'whats_changed', 'log'
-        ]
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json(old_gen=old_generation)
-            results = map(
-                lambda row:
-                    (row['generation'], row['doc_id'], row['transaction_id']),
-                response[2]['transactions'])
-            results.reverse()
-            cur_gen = old_generation
-            seen = set()
-            changes = []
-            newest_trans_id = ''
-            for generation, doc_id, trans_id in results:
-                if doc_id not in seen:
-                    changes.append((doc_id, generation, trans_id))
-                    seen.add(doc_id)
-            if changes:
-                cur_gen = changes[0][1]  # max generation
-                newest_trans_id = changes[0][2]
-                changes.reverse()
-            else:
-                cur_gen, newest_trans_id = self._get_generation_info()
-
-            return cur_gen, newest_trans_id, changes
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-        except ServerError as e:
-            raise_server_error(e, ddoc_path)
+        return self._database.whats_changed(old_generation)
 
     def delete_doc(self, doc):
         """
@@ -791,25 +480,6 @@ class CouchDatabase(CommonBackend):
         self._put_doc(old_doc, doc)
         return new_rev
 
-    def _build_conflicts(self, doc_id, attached_conflicts):
-        """
-        Build the conflicted documents list from the conflicts attachment
-        fetched from a couch document.
-
-        :param attached_conflicts: The document's conflicts as fetched from a
-                                   couch document attachment.
-        :type attached_conflicts: dict
-        """
-        conflicts = []
-        for doc_rev, content in attached_conflicts:
-            doc = self._factory(doc_id, doc_rev)
-            if content is None:
-                doc.make_tombstone()
-            else:
-                doc.content = content
-            conflicts.append(doc)
-        return conflicts
-
     def get_doc_conflicts(self, doc_id, couch_rev=None):
         """
         Get the conflicted versions of a document.
@@ -823,22 +493,7 @@ class CouchDatabase(CommonBackend):
         :return: A list of conflicted versions of the document.
         :rtype: list
         """
-        # request conflicts attachment from server
-        params = {}
-        conflicts = []
-        if couch_rev is not None:
-            params['rev'] = couch_rev  # restric document's couch revision
-        else:
-            # TODO: move into resource logic!
-            first_entry = self._get_doc(doc_id, check_for_conflicts=True)
-            conflicts.append(first_entry)
-        resource = self._database.resource(doc_id, 'u1db_conflicts')
-        try:
-            response = resource.get_json(**params)
-            return conflicts + self._build_conflicts(
-                doc_id, json.loads(response[2].read()))
-        except ResourceNotFound:
-            return []
+        return self._database.get_doc_conflicts(doc_id, couch_rev)
 
     def _get_replica_gen_and_trans_id(self, other_replica_uid):
         """
@@ -859,22 +514,7 @@ class CouchDatabase(CommonBackend):
                  synchronized with the replica, this is (0, '').
         :rtype: (int, str)
         """
-        if other_replica_uid in self.cache:
-            return self.cache[other_replica_uid]
-
-        doc_id = 'u1db_sync_%s' % other_replica_uid
-        try:
-            doc = self._database[doc_id]
-        except ResourceNotFound:
-            doc = {
-                '_id': doc_id,
-                'generation': 0,
-                'transaction_id': '',
-            }
-            self._database.save(doc)
-        result = doc['generation'], doc['transaction_id']
-        self.cache[other_replica_uid] = result
-        return result
+        return self._database._get_replica_gen_and_trans_id(other_replica_uid)
 
     def _set_replica_gen_and_trans_id(self, other_replica_uid,
                                       other_generation, other_transaction_id,
@@ -937,14 +577,9 @@ class CouchDatabase(CommonBackend):
         """
         self.cache[other_replica_uid] = (other_generation,
                                          other_transaction_id)
-        doc_id = 'u1db_sync_%s' % other_replica_uid
-        try:
-            doc = self._database[doc_id]
-        except ResourceNotFound:
-            doc = {'_id': doc_id}
-        doc['generation'] = other_generation
-        doc['transaction_id'] = other_transaction_id
-        self._database.save(doc)
+        self._database._do_set_replica_gen_and_trans_id(other_replica_uid,
+                                                        other_generation,
+                                                        other_transaction_id)
 
     def _force_doc_sync_conflict(self, doc):
         """
@@ -1099,25 +734,8 @@ class CouchDatabase(CommonBackend):
                  in matching doc_ids order.
         :rtype: iterable
         """
-        # Workaround for:
-        #
-        #   http://bugs.python.org/issue7980
-        #   https://leap.se/code/issues/5449
-        #
-        # python-couchdb uses time.strptime, which is not thread safe. In
-        # order to avoid the problem described on the issues above, we preload
-        # strptime here by evaluating the conversion of an arbitrary date.
-        # This will not be needed when/if we switch from python-couchdb to
-        # paisley.
-        time.strptime('Mar 8 1917', '%b %d %Y')
-        get_one = lambda doc_id: self._get_doc(doc_id, check_for_conflicts)
-        docs = [THREAD_POOL.apply_async(get_one, [doc_id])
-                for doc_id in doc_ids]
-        for doc in docs:
-            doc = doc.get()
-            if not doc or not include_deleted and doc.is_tombstone():
-                continue
-            yield doc
+        return self._database.get_docs(doc_ids, check_for_conflicts,
+                                       include_deleted)
 
     def _prune_conflicts(self, doc, doc_vcr):
         """
@@ -1147,6 +765,632 @@ class CouchDatabase(CommonBackend):
                 doc.rev = doc_vcr.as_str()
             doc.delete_conflicts(c_revs_to_prune)
 
+
+class CouchDatabase(object):
+    """
+    Holds CouchDB related code.
+    This class gives methods to encapsulate database operations and hide
+    CouchDB details from backend code.
+    """
+
+    @classmethod
+    def open_database(cls, url, create, ensure_ddocs=False):
+        """
+        Open a U1DB database using CouchDB as backend.
+
+        :param url: the url of the database replica
+        :type url: str
+        :param create: should the replica be created if it does not exist?
+        :type create: bool
+        :param replica_uid: an optional unique replica identifier
+        :type replica_uid: str
+        :param ensure_ddocs: Ensure that the design docs exist on server.
+        :type ensure_ddocs: bool
+
+        :return: the database instance
+        :rtype: SoledadBackend
+        """
+        # get database from url
+        m = re.match('(^https?://[^/]+)/(.+)$', url)
+        if not m:
+            raise InvalidURLError
+        url = m.group(1)
+        dbname = m.group(2)
+        with couch_server(url) as server:
+            try:
+                server[dbname]
+            except ResourceNotFound:
+                if not create:
+                    raise DatabaseDoesNotExist()
+                server.create(dbname)
+        return cls(
+            url, dbname, ensure_ddocs=ensure_ddocs)
+
+    def __init__(self, url, dbname, ensure_ddocs=True,
+                 database_security=None):
+        self._session = Session(timeout=COUCH_TIMEOUT)
+        self._url = url
+        self._dbname = dbname
+        self._database = Database(
+            urljoin(url, dbname),
+            self._session)
+        self._database.info()
+        if ensure_ddocs:
+            self.ensure_ddocs_on_db()
+            self.ensure_security_ddoc(database_security)
+
+    def ensure_ddocs_on_db(self):
+        """
+        Ensure that the design documents used by the backend exist on the
+        couch database.
+        """
+        for ddoc_name in ['docs', 'syncs', 'transactions']:
+            try:
+                self._database.resource('_design',
+                                        ddoc_name, '_info').get_json()
+            except ResourceNotFound:
+                ddoc = json.loads(
+                    binascii.a2b_base64(
+                        getattr(ddocs, ddoc_name)))
+                self._database.save(ddoc)
+
+    def ensure_security_ddoc(self, security_config=None):
+        """
+        Make sure that only soledad user is able to access this database as
+        an unprivileged member, meaning that administration access will
+        be forbidden even inside an user database.
+        The goal is to make sure that only the lowest access level is given
+        to the unprivileged CouchDB user set on the server process.
+        This is achieved by creating a _security design document, see:
+        http://docs.couchdb.org/en/latest/api/database/security.html
+
+        :param database_security: security configuration parsed from conf file
+        :type cache: dict
+        """
+        security_config = security_config or {}
+        security = self._database.resource.get_json('_security')[2]
+        security['members'] = {'names': [], 'roles': []}
+        security['members']['names'] = security_config.get('members',
+                                                           ['soledad'])
+        security['members']['roles'] = security_config.get('members_roles', [])
+        security['admins'] = {'names': [], 'roles': []}
+        security['admins']['names'] = security_config.get('admins', [])
+        security['admins']['roles'] = security_config.get('admins_roles', [])
+        self._database.resource.put_json('_security', body=security)
+
+    def delete_database(self):
+        """
+        Delete a U1DB CouchDB database.
+        """
+        with couch_server(self._url) as server:
+            del(server[self._dbname])
+
+    def set_replica_uid(self, replica_uid):
+        """
+        Force the replica uid to be set.
+
+        :param replica_uid: The new replica uid.
+        :type replica_uid: str
+        """
+        try:
+            # set on existent config document
+            doc = self._database['u1db_config']
+            doc['replica_uid'] = replica_uid
+        except ResourceNotFound:
+            # or create the config document
+            doc = {
+                '_id': 'u1db_config',
+                'replica_uid': replica_uid,
+            }
+        self._database.save(doc)
+
+    def get_replica_uid(self):
+        """
+        Get the replica uid.
+
+        :return: The replica uid.
+        :rtype: str
+        """
+        try:
+            # grab replica_uid from server
+            doc = self._database['u1db_config']
+            replica_uid = doc['replica_uid']
+            return replica_uid
+        except ResourceNotFound:
+            # create a unique replica_uid
+            replica_uid = uuid.uuid4().hex
+            self.set_replica_uid(replica_uid)
+            return replica_uid
+
+    def close(self):
+        self._database = None
+
+    def get_all_docs(self, include_deleted=False):
+        """
+        Get the JSON content for all documents in the database.
+
+        :param include_deleted: If set to True, deleted documents will be
+                                returned with empty content. Otherwise deleted
+                                documents will not be included in the results.
+        :type include_deleted: bool
+
+        :return: (generation, [CouchDocument])
+            The current generation of the database, followed by a list of all
+            the documents in the database.
+        :rtype: (int, [CouchDocument])
+        """
+
+        generation, _ = self._get_generation_info()
+        results = list(self.get_docs(self._database,
+                                     include_deleted=include_deleted))
+        return (generation, results)
+
+    def get_docs(self, doc_ids, check_for_conflicts=True,
+                 include_deleted=False):
+        """
+        Get the JSON content for many documents.
+
+        :param doc_ids: A list of document identifiers or None for all.
+        :type doc_ids: list
+        :param check_for_conflicts: If set to False, then the conflict check
+                                    will be skipped, and 'None' will be
+                                    returned instead of True/False.
+        :type check_for_conflicts: bool
+        :param include_deleted: If set to True, deleted documents will be
+                                returned with empty content. Otherwise deleted
+                                documents will not be included in the results.
+        :return: iterable giving the Document object for each document id
+                 in matching doc_ids order.
+        :rtype: iterable
+        """
+        # Workaround for:
+        #
+        #   http://bugs.python.org/issue7980
+        #   https://leap.se/code/issues/5449
+        #
+        # python-couchdb uses time.strptime, which is not thread safe. In
+        # order to avoid the problem described on the issues above, we preload
+        # strptime here by evaluating the conversion of an arbitrary date.
+        # This will not be needed when/if we switch from python-couchdb to
+        # paisley.
+        time.strptime('Mar 8 1917', '%b %d %Y')
+        get_one = lambda doc_id: self._get_doc(doc_id, check_for_conflicts)
+        docs = [THREAD_POOL.apply_async(get_one, [doc_id])
+                for doc_id in doc_ids]
+        for doc in docs:
+            doc = doc.get()
+            if not doc or not include_deleted and doc.is_tombstone():
+                continue
+            yield doc
+
+    def _get_doc(self, doc_id, check_for_conflicts=False):
+        """
+        Extract the document from storage.
+
+        This can return None if the document doesn't exist.
+
+        :param doc_id: The unique document identifier
+        :type doc_id: str
+        :param check_for_conflicts: If set to False, then the conflict check
+                                    will be skipped.
+        :type check_for_conflicts: bool
+
+        :return: The document.
+        :rtype: CouchDocument
+        """
+        # get document with all attachments (u1db content and eventual
+        # conflicts)
+        try:
+            result = \
+                self._database.resource(doc_id).get_json(
+                    attachments=True)[2]
+        except ResourceNotFound:
+            return None
+        return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts)
+
+    def __parse_doc_from_couch(self, result, doc_id,
+                               check_for_conflicts=False):
+        # restrict to u1db documents
+        if 'u1db_rev' not in result:
+            return None
+        doc = CouchDocument(doc_id, result['u1db_rev'])
+        # set contents or make tombstone
+        if '_attachments' not in result \
+                or 'u1db_content' not in result['_attachments']:
+            doc.make_tombstone()
+        else:
+            doc.content = json.loads(
+                binascii.a2b_base64(
+                    result['_attachments']['u1db_content']['data']))
+        # determine if there are conflicts
+        if check_for_conflicts \
+                and '_attachments' in result \
+                and 'u1db_conflicts' in result['_attachments']:
+            doc.set_conflicts(
+                self._build_conflicts(
+                    doc.doc_id,
+                    json.loads(binascii.a2b_base64(
+                        result['_attachments']['u1db_conflicts']['data']))))
+        # store couch revision
+        doc.couch_rev = result['_rev']
+        # store transactions
+        doc.transactions = result['u1db_transactions']
+        return doc
+
+    def _build_conflicts(self, doc_id, attached_conflicts):
+        """
+        Build the conflicted documents list from the conflicts attachment
+        fetched from a couch document.
+
+        :param attached_conflicts: The document's conflicts as fetched from a
+                                   couch document attachment.
+        :type attached_conflicts: dict
+        """
+        conflicts = []
+        for doc_rev, content in attached_conflicts:
+            doc = CouchDocument(doc_id, doc_rev)
+            if content is None:
+                doc.make_tombstone()
+            else:
+                doc.content = content
+            conflicts.append(doc)
+        return conflicts
+
+    def _get_trans_id_for_gen(self, generation):
+        """
+        Get the transaction id corresponding to a particular generation.
+
+        :param generation: The generation for which to get the transaction id.
+        :type generation: int
+
+        :return: The transaction id for C{generation}.
+        :rtype: str
+
+        :raise InvalidGeneration: Raised when the generation does not exist.
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        if generation == 0:
+            return ''
+        # query a couch list function
+        ddoc_path = [
+            '_design', 'transactions', '_list', 'trans_id_for_gen', 'log'
+        ]
+        res = self._database.resource(*ddoc_path)
+        try:
+            response = res.get_json(gen=generation)
+            if response[2] == {}:
+                raise InvalidGeneration
+            return response[2]['transaction_id']
+        except ResourceNotFound as e:
+            raise_missing_design_doc_error(e, ddoc_path)
+        except ServerError as e:
+            raise_server_error(e, ddoc_path)
+
+    def _get_replica_gen_and_trans_id(self, other_replica_uid):
+        """
+        Return the last known generation and transaction id for the other db
+        replica.
+
+        When you do a synchronization with another replica, the Database keeps
+        track of what generation the other database replica was at, and what
+        the associated transaction id was.  This is used to determine what data
+        needs to be sent, and if two databases are claiming to be the same
+        replica.
+
+        :param other_replica_uid: The identifier for the other replica.
+        :type other_replica_uid: str
+
+        :return: A tuple containing the generation and transaction id we
+                 encountered during synchronization. If we've never
+                 synchronized with the replica, this is (0, '').
+        :rtype: (int, str)
+        """
+        doc_id = 'u1db_sync_%s' % other_replica_uid
+        try:
+            doc = self._database[doc_id]
+        except ResourceNotFound:
+            doc = {
+                '_id': doc_id,
+                'generation': 0,
+                'transaction_id': '',
+            }
+            self._database.save(doc)
+        result = doc['generation'], doc['transaction_id']
+        return result
+
+    def get_doc_conflicts(self, doc_id, couch_rev=None):
+        """
+        Get the conflicted versions of a document.
+
+        If the C{couch_rev} parameter is not None, conflicts for a specific
+        document's couch revision are returned.
+
+        :param couch_rev: The couch document revision.
+        :type couch_rev: str
+
+        :return: A list of conflicted versions of the document.
+        :rtype: list
+        """
+        # request conflicts attachment from server
+        params = {}
+        conflicts = []
+        if couch_rev is not None:
+            params['rev'] = couch_rev  # restric document's couch revision
+        else:
+            # TODO: move into resource logic!
+            first_entry = self._get_doc(doc_id, check_for_conflicts=True)
+            conflicts.append(first_entry)
+        resource = self._database.resource(doc_id, 'u1db_conflicts')
+        try:
+            response = resource.get_json(**params)
+            return conflicts + self._build_conflicts(
+                doc_id, json.loads(response[2].read()))
+        except ResourceNotFound:
+            return []
+
+    def _do_set_replica_gen_and_trans_id(
+            self, other_replica_uid, other_generation, other_transaction_id,
+            number_of_docs=None, doc_idx=None, sync_id=None):
+        """
+        Set the last-known generation and transaction id for the other
+        database replica.
+
+        We have just performed some synchronization, and we want to track what
+        generation the other replica was at. See also
+        _get_replica_gen_and_trans_id.
+
+        :param other_replica_uid: The U1DB identifier for the other replica.
+        :type other_replica_uid: str
+        :param other_generation: The generation number for the other replica.
+        :type other_generation: int
+        :param other_transaction_id: The transaction id associated with the
+                                     generation.
+        :type other_transaction_id: str
+        :param number_of_docs: The total amount of documents sent on this sync
+                               session.
+        :type number_of_docs: int
+        :param doc_idx: The index of the current document being sent.
+        :type doc_idx: int
+        :param sync_id: The id of the current sync session.
+        :type sync_id: str
+        """
+        doc_id = 'u1db_sync_%s' % other_replica_uid
+        try:
+            doc = self._database[doc_id]
+        except ResourceNotFound:
+            doc = {'_id': doc_id}
+        doc['generation'] = other_generation
+        doc['transaction_id'] = other_transaction_id
+        self._database.save(doc)
+
+    def _get_transaction_log(self):
+        """
+        This is only for the test suite, it is not part of the api.
+
+        :return: The complete transaction log.
+        :rtype: [(str, str)]
+
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        # query a couch view
+        ddoc_path = ['_design', 'transactions', '_view', 'log']
+        res = self._database.resource(*ddoc_path)
+        try:
+            response = res.get_json()
+            return map(
+                lambda row: (row['id'], row['value']),
+                response[2]['rows'])
+        except ResourceNotFound as e:
+            raise_missing_design_doc_error(e, ddoc_path)
+
+    def whats_changed(self, old_generation=0):
+        """
+        Return a list of documents that have changed since old_generation.
+
+        :param old_generation: The generation of the database in the old
+                               state.
+        :type old_generation: int
+
+        :return: (generation, trans_id, [(doc_id, generation, trans_id),...])
+                 The current generation of the database, its associated
+                 transaction id, and a list of of changed documents since
+                 old_generation, represented by tuples with for each document
+                 its doc_id and the generation and transaction id corresponding
+                 to the last intervening change and sorted by generation (old
+                 changes first)
+        :rtype: (int, str, [(str, int, str)])
+
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        # query a couch list function
+        ddoc_path = [
+            '_design', 'transactions', '_list', 'whats_changed', 'log'
+        ]
+        res = self._database.resource(*ddoc_path)
+        try:
+            response = res.get_json(old_gen=old_generation)
+            results = map(
+                lambda row:
+                    (row['generation'], row['doc_id'], row['transaction_id']),
+                response[2]['transactions'])
+            results.reverse()
+            cur_gen = old_generation
+            seen = set()
+            changes = []
+            newest_trans_id = ''
+            for generation, doc_id, trans_id in results:
+                if doc_id not in seen:
+                    changes.append((doc_id, generation, trans_id))
+                    seen.add(doc_id)
+            if changes:
+                cur_gen = changes[0][1]  # max generation
+                newest_trans_id = changes[0][2]
+                changes.reverse()
+            else:
+                cur_gen, newest_trans_id = self._get_generation_info()
+
+            return cur_gen, newest_trans_id, changes
+        except ResourceNotFound as e:
+            raise_missing_design_doc_error(e, ddoc_path)
+        except ServerError as e:
+            raise_server_error(e, ddoc_path)
+
+    def _get_generation_info(self):
+        """
+        Return the current generation.
+
+        :return: A tuple containing the current generation and transaction id.
+        :rtype: (int, str)
+
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        # query a couch list function
+        ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
+        res = self._database.resource(*ddoc_path)
+        try:
+            response = res.get_json()
+            return (response[2]['generation'], response[2]['transaction_id'])
+        except ResourceNotFound as e:
+            raise_missing_design_doc_error(e, ddoc_path)
+        except ServerError as e:
+            raise_server_error(e, ddoc_path)
+
+    def save_document(self, old_doc, doc, transaction_id):
+        """
+        Put the document in the Couch backend database.
+
+        Note that C{old_doc} must have been fetched with the parameter
+        C{check_for_conflicts} equal to True, so we can properly update the
+        new document using the conflict information from the old one.
+
+        :param old_doc: The old document version.
+        :type old_doc: CouchDocument
+        :param doc: The document to be put.
+        :type doc: CouchDocument
+
+        :raise RevisionConflict: Raised when trying to update a document but
+                                 couch revisions mismatch.
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        attachments = {}  # we save content and conflicts as attachments
+        parts = []  # and we put it using couch's multipart PUT
+        # save content as attachment
+        if doc.is_tombstone() is False:
+            content = doc.get_json()
+            attachments['u1db_content'] = {
+                'follows': True,
+                'content_type': 'application/octet-stream',
+                'length': len(content),
+            }
+            parts.append(content)
+        # save conflicts as attachment
+        if doc.has_conflicts is True:
+            conflicts = json.dumps(
+                map(lambda cdoc: (cdoc.rev, cdoc.content),
+                    doc.get_conflicts()))
+            attachments['u1db_conflicts'] = {
+                'follows': True,
+                'content_type': 'application/octet-stream',
+                'length': len(conflicts),
+            }
+            parts.append(conflicts)
+        # store old transactions, if any
+        transactions = old_doc.transactions[:] if old_doc is not None else []
+        # create a new transaction id and timestamp it so the transaction log
+        # is consistent when querying the database.
+        transactions.append(
+            # here we store milliseconds to keep consistent with javascript
+            # Date.prototype.getTime() which was used before inside a couchdb
+            # update handler.
+            (int(time.time() * 1000),
+             transaction_id))
+        # build the couch document
+        couch_doc = {
+            '_id': doc.doc_id,
+            'u1db_rev': doc.rev,
+            'u1db_transactions': transactions,
+            '_attachments': attachments,
+        }
+        # if we are updating a doc we have to add the couch doc revision
+        if old_doc is not None:
+            couch_doc['_rev'] = old_doc.couch_rev
+        # prepare the multipart PUT
+        buf = StringIO()
+        headers = {}
+        envelope = MultipartWriter(buf, headers=headers, subtype='related')
+        envelope.add('application/json', json.dumps(couch_doc))
+        for part in parts:
+            envelope.add('application/octet-stream', part)
+        envelope.close()
+        # try to save and fail if there's a revision conflict
+        try:
+            resource = self._new_resource()
+            resource.put_json(
+                doc.doc_id, body=str(buf.getvalue()), headers=headers)
+        except ResourceConflict:
+            raise RevisionConflict()
+        return transactions[-1][1]
+
     def _new_resource(self, *path):
         """
         Return a new resource for accessing a couch database.
@@ -1165,7 +1409,7 @@ class CouchDatabase(CommonBackend):
 class CouchSyncTarget(CommonSyncTarget):
 
     """
-    Functionality for using a CouchDatabase as a synchronization target.
+    Functionality for using a SoledadBackend as a synchronization target.
     """
 
     def get_sync_info(self, source_replica_uid):
@@ -1222,13 +1466,12 @@ class CouchServerState(ServerState):
         :param dbname: The name of the database to open.
         :type dbname: str
 
-        :return: The CouchDatabase object.
-        :rtype: CouchDatabase
+        :return: The SoledadBackend object.
+        :rtype: SoledadBackend
         """
-        db = CouchDatabase(
-            self.couch_url,
-            dbname,
-            ensure_ddocs=False)
+        url = urljoin(self.couch_url, dbname)
+        db = SoledadBackend.open_database(url, create=False,
+                                          ensure_ddocs=False)
         return db
 
     def ensure_database(self, dbname):
@@ -1240,8 +1483,8 @@ class CouchServerState(ServerState):
 
         :raise Unauthorized: If disabled or other error was raised.
 
-        :return: The CouchDatabase object and its replica_uid.
-        :rtype: (CouchDatabase, str)
+        :return: The SoledadBackend object and its replica_uid.
+        :rtype: (SoledadBackend, str)
         """
         if not self.create_cmd:
             raise Unauthorized()
diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py
index 2ba05839..ea9d6ce4 100644
--- a/common/src/leap/soledad/common/errors.py
+++ b/common/src/leap/soledad/common/errors.py
@@ -132,7 +132,7 @@ class CouldNotObtainLockError(SoledadError):
 
 
 #
-# CouchDatabase errors
+# SoledadBackend errors
 #
 
 @register_exception
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 86cc0881..b3536a6a 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -56,7 +56,7 @@ from u1db.backends.inmemory import InMemoryIndex
 class TestCouchBackendImpl(CouchDBTestCase):
 
     def test__allocate_doc_id(self):
-        db = couch.CouchDatabase.open_database(
+        db = couch.SoledadBackend.open_database(
             urljoin(
                 'http://localhost:' + str(self.couch_port),
                 ('test-%s' % uuid4().hex)
@@ -78,7 +78,7 @@ class TestCouchBackendImpl(CouchDBTestCase):
 def make_couch_database_for_test(test, replica_uid):
     port = str(test.couch_port)
     dbname = ('test-%s' % uuid4().hex)
-    db = couch.CouchDatabase.open_database(
+    db = couch.SoledadBackend.open_database(
         urljoin('http://localhost:' + port, dbname),
         create=True,
         replica_uid=replica_uid or 'test',
@@ -91,7 +91,7 @@ def copy_couch_database_for_test(test, db):
     port = str(test.couch_port)
     couch_url = 'http://localhost:' + port
     new_dbname = db._dbname + '_copy'
-    new_db = couch.CouchDatabase.open_database(
+    new_db = couch.SoledadBackend.open_database(
         urljoin(couch_url, new_dbname),
         create=True,
         replica_uid=db._replica_uid or 'test')
@@ -150,7 +150,7 @@ class CouchTests(
     scenarios = COUCH_SCENARIOS
 
 
-class CouchDatabaseTests(
+class SoledadBackendTests(
         TestWithScenarios,
         test_backends.LocalDatabaseTests,
         CouchDBTestCase):
@@ -206,7 +206,7 @@ simple_doc = tests.simple_doc
 nested_doc = tests.nested_doc
 
 
-class CouchDatabaseSyncTargetTests(
+class SoledadBackendSyncTargetTests(
         TestWithScenarios,
         DatabaseBaseTests,
         CouchDBTestCase):
@@ -532,13 +532,10 @@ class CouchDatabaseSyncTargetTests(
 
 # The following tests need that the database have an index, so we fake one.
 
-class IndexedCouchDatabase(couch.CouchDatabase):
+class IndexedSoledadBackend(couch.SoledadBackend):
 
-    def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True,
-                 database_security=None):
-        old_class.__init__(self, url, dbname, replica_uid=replica_uid,
-                           ensure_ddocs=ensure_ddocs,
-                           database_security=database_security)
+    def __init__(self, db, replica_uid=None):
+        old_class.__init__(self, db, replica_uid=replica_uid)
         self._indexes = {}
 
     def _put_doc(self, old_doc, doc):
@@ -608,11 +605,11 @@ class IndexedCouchDatabase(couch.CouchDatabase):
         return list(set([tuple(key.split('\x01')) for key in keys]))
 
 
-# monkey patch CouchDatabase (once) to include virtual indexes
-if getattr(couch.CouchDatabase, '_old_class', None) is None:
-    old_class = couch.CouchDatabase
-    IndexedCouchDatabase._old_class = old_class
-    couch.CouchDatabase = IndexedCouchDatabase
+# monkey patch SoledadBackend (once) to include virtual indexes
+if getattr(couch.SoledadBackend, '_old_class', None) is None:
+    old_class = couch.SoledadBackend
+    IndexedSoledadBackend._old_class = old_class
+    couch.SoledadBackend = IndexedSoledadBackend
 
 
 sync_scenarios = []
@@ -623,7 +620,7 @@ for name, scenario in COUCH_SCENARIOS:
     scenario = dict(scenario)
 
 
-class CouchDatabaseSyncTests(
+class SoledadBackendSyncTests(
         TestWithScenarios,
         DatabaseBaseTests,
         CouchDBTestCase):
@@ -1319,7 +1316,7 @@ class CouchDatabaseSyncTests(
         self.assertEqual(cont2, self.db1.get_doc("2").get_json())
 
 
-class CouchDatabaseExceptionsTests(CouchDBTestCase):
+class SoledadBackendExceptionsTests(CouchDBTestCase):
 
     def setUp(self):
         CouchDBTestCase.setUp(self)
@@ -1343,10 +1340,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         design docs are not present.
         """
         self.create_db(ensure=False)
-        # _get_generation()
-        self.assertRaises(
-            errors.MissingDesignDocError,
-            self.db._get_generation)
         # _get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocError,
@@ -1374,10 +1367,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         transactions = self.db._database['_design/transactions']
         transactions['lists'] = {}
         self.db._database.save(transactions)
-        # _get_generation()
-        self.assertRaises(
-            errors.MissingDesignDocListFunctionError,
-            self.db._get_generation)
         # _get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocListFunctionError,
@@ -1401,10 +1390,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         transactions = self.db._database['_design/transactions']
         del transactions['lists']
         self.db._database.save(transactions)
-        # _get_generation()
-        self.assertRaises(
-            errors.MissingDesignDocListFunctionError,
-            self.db._get_generation)
         # _get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocListFunctionError,
@@ -1436,10 +1421,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         transactions = self.db._database['_design/transactions']
         del transactions['views']
         self.db._database.save(transactions)
-        # _get_generation()
-        self.assertRaises(
-            errors.MissingDesignDocNamedViewError,
-            self.db._get_generation)
         # _get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocNamedViewError,
@@ -1469,10 +1450,6 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase):
         del self.db._database['_design/syncs']
         # delete _design/transactions
         del self.db._database['_design/transactions']
-        # _get_generation()
-        self.assertRaises(
-            errors.MissingDesignDocDeletedError,
-            self.db._get_generation)
         # _get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocDeletedError,
diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
index 507f2984..2c2daf05 100644
--- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
+++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
@@ -26,7 +26,7 @@ from twisted.internet import defer
 from uuid import uuid4
 
 from leap.soledad.client import Soledad
-from leap.soledad.common.couch import CouchDatabase, CouchServerState
+from leap.soledad.common.couch import SoledadBackend, CouchServerState
 
 from leap.soledad.common.tests.util import (
     make_token_soledad_app,
@@ -86,7 +86,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
         TestCaseWithServer.setUp(self)
         CouchDBTestCase.setUp(self)
         self.user = ('user-%s' % uuid4().hex)
-        self.db = CouchDatabase.open_database(
+        self.db = SoledadBackend.open_database(
             urljoin(self.couch_url, 'user-' + self.user),
             create=True,
             replica_uid='replica',
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index 7c006121..5810ce31 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -31,7 +31,7 @@ from twisted.trial import unittest
 
 from leap.soledad.common.couch import (
     CouchServerState,
-    CouchDatabase,
+    SoledadBackend,
 )
 from leap.soledad.common.tests.u1db_tests import TestCaseWithServer
 from leap.soledad.common.tests.test_couch import CouchDBTestCase
@@ -358,7 +358,7 @@ class EncryptedSyncTestCase(
             passphrase=passphrase)
 
         # ensure remote db exists before syncing
-        db = CouchDatabase.open_database(
+        db = SoledadBackend.open_database(
             urljoin(self.couch_url, 'user-' + user),
             create=True,
             ensure_ddocs=True)
@@ -488,7 +488,7 @@ class LockResourceTestCase(
         self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
         TestCaseWithServer.setUp(self)
         # create the databases
-        db = CouchDatabase.open_database(
+        db = SoledadBackend.open_database(
             urljoin(self.couch_url, ('shared-%s' % (uuid4().hex))),
             create=True,
             ensure_ddocs=True)
diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py
index 1041367b..88443179 100644
--- a/common/src/leap/soledad/common/tests/test_sync.py
+++ b/common/src/leap/soledad/common/tests/test_sync.py
@@ -101,7 +101,7 @@ class InterruptableSyncTestCase(
             user='user-uuid', server_url=self.getURL())
 
         # ensure remote db exists before syncing
-        db = couch.CouchDatabase.open_database(
+        db = couch.SoledadBackend.open_database(
             urljoin(self.couch_url, 'user-user-uuid'),
             create=True,
             ensure_ddocs=True)
diff --git a/common/src/leap/soledad/common/tests/test_sync_mutex.py b/common/src/leap/soledad/common/tests/test_sync_mutex.py
index 2e2123a7..35cecb8f 100644
--- a/common/src/leap/soledad/common/tests/test_sync_mutex.py
+++ b/common/src/leap/soledad/common/tests/test_sync_mutex.py
@@ -102,7 +102,7 @@ class TestSyncMutex(
         self.startServer()
 
         # ensure remote db exists before syncing
-        db = couch.CouchDatabase.open_database(
+        db = couch.SoledadBackend.open_database(
             urljoin(self.couch_url, 'user-' + self.user),
             create=True,
             ensure_ddocs=True)
diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py
index c0987e90..0cf31c3f 100644
--- a/common/src/leap/soledad/common/tests/test_sync_target.py
+++ b/common/src/leap/soledad/common/tests/test_sync_target.py
@@ -265,9 +265,9 @@ class TestSoledadSyncTarget(
                                      replica_trans_id=replica_trans_id,
                                      number_of_docs=number_of_docs,
                                      doc_idx=doc_idx, sync_id=sync_id)
-        from leap.soledad.common.tests.test_couch import IndexedCouchDatabase
+        from leap.soledad.common.tests.test_couch import IndexedSoledadBackend
         self.patch(
-            IndexedCouchDatabase, '_put_doc_if_newer', bomb_put_doc_if_newer)
+            IndexedSoledadBackend, '_put_doc_if_newer', bomb_put_doc_if_newer)
         remote_target = self.getSyncTarget(
             source_replica_uid='replica')
         other_changes = []
diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py
index 1c7adb91..bfd06856 100644
--- a/common/src/leap/soledad/common/tests/util.py
+++ b/common/src/leap/soledad/common/tests/util.py
@@ -47,7 +47,7 @@ from leap.common.testing.basetest import BaseLeapTest
 
 from leap.soledad.common import soledad_assert
 from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common.couch import CouchDatabase, CouchServerState
+from leap.soledad.common.couch import SoledadBackend, CouchServerState
 from leap.soledad.common.crypto import ENC_SCHEME_KEY
 
 from leap.soledad.client import Soledad
@@ -379,7 +379,7 @@ class CouchServerStateForTests(CouchServerState):
         Create db and append to a list, allowing test to close it later
         """
         dbname = dbname or ('test-%s' % uuid4().hex)
-        db = CouchDatabase.open_database(
+        db = SoledadBackend.open_database(
             urljoin(self.couch_url, dbname),
             True,
             replica_uid=replica_uid or 'test',
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index ae5d15dc..a8ba3833 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -19,7 +19,7 @@ import os
 import sys
 import netrc
 import argparse
-from leap.soledad.common.couch import CouchDatabase
+from leap.soledad.common.couch import SoledadBackend
 from leap.soledad.common.couch import is_db_name_valid
 from leap.soledad.common.couch import list_users_dbs
 from leap.soledad.server import load_configuration
@@ -69,7 +69,7 @@ def ensure_database(dbname):
         sys.exit(1)
     url = url_for_db(dbname)
     db_security = CONF['database-security']
-    db = CouchDatabase.open_database(url=url, create=True,
+    db = SoledadBackend.open_database(url=url, create=True,
                                      replica_uid=None, ensure_ddocs=True,
                                      database_security=db_security)
     print ('success! Ensured that database %s exists, with replica_uid: %s' %
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index 92b29102..db25c406 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -32,7 +32,7 @@ class SyncExchange(sync.SyncExchange):
     def __init__(self, db, source_replica_uid, last_known_generation, sync_id):
         """
         :param db: The target syncing database.
-        :type db: CouchDatabase
+        :type db: SoledadBackend
         :param source_replica_uid: The uid of the source syncing replica.
         :type source_replica_uid: str
         :param last_known_generation: The last target replica generation the
-- 
cgit v1.2.3


From f0b96af943dcb6c8cde4f6d4280186d78c78096c Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 13 Oct 2015 21:34:40 -0300
Subject: [refactor] split out backend from couch database

First step of splitting classes across files on common.
backend.py holds SoledadBackend (generic backend logic)
couch/ is now a directory with old code inside __init__.py and
CouchServerState on state.py
Also removed mock IndexedSoledadBackend, since Soledad does not support
indexing due to encryption on server side.
Also fixed DesignDocUnknownError to show up what is the message of the
original exception. It was being lost.
---
 common/src/leap/soledad/common/backend.py          |  681 +++++++++
 common/src/leap/soledad/common/couch.py            | 1514 --------------------
 common/src/leap/soledad/common/couch/__init__.py   |  746 ++++++++++
 common/src/leap/soledad/common/couch/state.py      |  114 ++
 common/src/leap/soledad/common/errors.py           |    2 +-
 common/src/leap/soledad/common/tests/test_couch.py |  149 +-
 .../tests/test_couch_operations_atomicity.py       |    5 +-
 .../src/leap/soledad/common/tests/test_server.py   |   10 +-
 .../leap/soledad/common/tests/test_sync_mutex.py   |    7 +-
 .../leap/soledad/common/tests/test_sync_target.py  |    6 +-
 common/src/leap/soledad/common/tests/util.py       |    9 +-
 server/pkg/create-user-db                          |    6 +-
 server/src/leap/soledad/server/__init__.py         |    2 +-
 13 files changed, 1597 insertions(+), 1654 deletions(-)
 create mode 100644 common/src/leap/soledad/common/backend.py
 delete mode 100644 common/src/leap/soledad/common/couch.py
 create mode 100644 common/src/leap/soledad/common/couch/__init__.py
 create mode 100644 common/src/leap/soledad/common/couch/state.py

diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
new file mode 100644
index 00000000..cb37b4ac
--- /dev/null
+++ b/common/src/leap/soledad/common/backend.py
@@ -0,0 +1,681 @@
+# -*- coding: utf-8 -*-
+# backend.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""A U1DB generic backend."""
+
+
+from u1db import vectorclock
+from u1db.errors import (
+    RevisionConflict,
+    InvalidDocId,
+    ConflictedDoc,
+    DocumentDoesNotExist,
+    DocumentAlreadyDeleted,
+)
+from u1db.backends import CommonBackend
+from u1db.backends import CommonSyncTarget
+from leap.soledad.common.document import CouchDocument
+
+
+class SoledadBackend(CommonBackend):
+
+    """
+    A U1DB backend implementation.
+    """
+
+    def __init__(self, database, replica_uid=None):
+        """
+        Create a new backend.
+
+        :param database: the database implementation
+        :type database: Database
+        :param replica_uid: an optional unique replica identifier
+        :type replica_uid: str
+        """
+        # save params
+        self._factory = CouchDocument
+        self._real_replica_uid = None
+        self._cache = None
+        self._dbname = database._dbname
+        self._database = database
+        if replica_uid is not None:
+            self._set_replica_uid(replica_uid)
+
+    @property
+    def cache(self):
+        if self._cache is not None:
+            return self._cache
+        else:
+            return {}
+
+    def init_caching(self, cache):
+        """
+        Start using cache by setting internal _cache attribute.
+
+        :param cache: the cache instance, anything that behaves like a dict
+        :type cache: dict
+        """
+        self._cache = cache
+
+    def get_sync_target(self):
+        """
+        Return a SyncTarget object, for another u1db to synchronize with.
+
+        :return: The sync target.
+        :rtype: SoledadSyncTarget
+        """
+        return SoledadSyncTarget(self)
+
+    def delete_database(self):
+        """
+        Delete a U1DB CouchDB database.
+        """
+        self._database.delete_database()
+
+    def close(self):
+        """
+        Release any resources associated with this database.
+
+        :return: True if db was succesfully closed.
+        :rtype: bool
+        """
+        self._database.close()
+        return True
+
+    def __del__(self):
+        """
+        Close the database upon garbage collection.
+        """
+        self.close()
+
+    def _set_replica_uid(self, replica_uid):
+        """
+        Force the replica uid to be set.
+
+        :param replica_uid: The new replica uid.
+        :type replica_uid: str
+        """
+        self._database.set_replica_uid(replica_uid)
+        self._real_replica_uid = replica_uid
+        self.cache['replica_uid'] = self._real_replica_uid
+
+    def _get_replica_uid(self):
+        """
+        Get the replica uid.
+
+        :return: The replica uid.
+        :rtype: str
+        """
+        if self._real_replica_uid is not None:
+            self.cache['replica_uid'] = self._real_replica_uid
+            return self._real_replica_uid
+        if 'replica_uid' in self.cache:
+            return self.cache['replica_uid']
+        self._real_replica_uid = self._database.get_replica_uid()
+        self._set_replica_uid(self._real_replica_uid)
+        return self._real_replica_uid
+
+    _replica_uid = property(_get_replica_uid, _set_replica_uid)
+
+    replica_uid = property(_get_replica_uid)
+
+    def _get_generation(self):
+        """
+        Return the current generation.
+
+        :return: The current generation.
+        :rtype: int
+
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        return self._get_generation_info()[0]
+
+    def _get_generation_info(self):
+        """
+        Return the current generation.
+
+        :return: A tuple containing the current generation and transaction id.
+        :rtype: (int, str)
+
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        if self.replica_uid + '_gen' in self.cache:
+            response = self.cache[self.replica_uid + '_gen']
+            return response
+        cur_gen, newest_trans_id = self._database._get_generation_info()
+        self.cache[self.replica_uid + '_gen'] = (cur_gen, newest_trans_id)
+        return (cur_gen, newest_trans_id)
+
+    def _get_trans_id_for_gen(self, generation):
+        """
+        Get the transaction id corresponding to a particular generation.
+
+        :param generation: The generation for which to get the transaction id.
+        :type generation: int
+
+        :return: The transaction id for C{generation}.
+        :rtype: str
+
+        """
+        return self._database._get_trans_id_for_gen(generation)
+
+    def _get_transaction_log(self):
+        """
+        This is only for the test suite, it is not part of the api.
+
+        :return: The complete transaction log.
+        :rtype: [(str, str)]
+
+        """
+        return self._database._get_transaction_log()
+
+    def _get_doc(self, doc_id, check_for_conflicts=False):
+        """
+        Extract the document from storage.
+
+        This can return None if the document doesn't exist.
+
+        :param doc_id: The unique document identifier
+        :type doc_id: str
+        :param check_for_conflicts: If set to False, then the conflict check
+                                    will be skipped.
+        :type check_for_conflicts: bool
+
+        :return: The document.
+        :rtype: CouchDocument
+        """
+        return self._database._get_doc(doc_id, check_for_conflicts)
+
+    def get_doc(self, doc_id, include_deleted=False):
+        """
+        Get the JSON string for the given document.
+
+        :param doc_id: The unique document identifier
+        :type doc_id: str
+        :param include_deleted: If set to True, deleted documents will be
+            returned with empty content. Otherwise asking for a deleted
+            document will return None.
+        :type include_deleted: bool
+
+        :return: A document object.
+        :rtype: CouchDocument.
+        """
+        doc = self._get_doc(doc_id, check_for_conflicts=True)
+        if doc is None:
+            return None
+        if doc.is_tombstone() and not include_deleted:
+            return None
+        return doc
+
+    def get_all_docs(self, include_deleted=False):
+        """
+        Get the JSON content for all documents in the database.
+
+        :param include_deleted: If set to True, deleted documents will be
+                                returned with empty content. Otherwise deleted
+                                documents will not be included in the results.
+        :type include_deleted: bool
+
+        :return: (generation, [CouchDocument])
+            The current generation of the database, followed by a list of all
+            the documents in the database.
+        :rtype: (int, [CouchDocument])
+        """
+        return self._database.get_all_docs(include_deleted)
+
+    def _put_doc(self, old_doc, doc):
+        """
+        Put the document in the Couch backend database.
+
+        Note that C{old_doc} must have been fetched with the parameter
+        C{check_for_conflicts} equal to True, so we can properly update the
+        new document using the conflict information from the old one.
+
+        :param old_doc: The old document version.
+        :type old_doc: CouchDocument
+        :param doc: The document to be put.
+        :type doc: CouchDocument
+        """
+        last_transaction =\
+            self._database.save_document(old_doc, doc,
+                                         self._allocate_transaction_id())
+        if self.replica_uid + '_gen' in self.cache:
+            gen, trans = self.cache[self.replica_uid + '_gen']
+            gen += 1
+            trans = last_transaction
+            self.cache[self.replica_uid + '_gen'] = (gen, trans)
+
+    def put_doc(self, doc):
+        """
+        Update a document.
+
+        If the document currently has conflicts, put will fail.
+        If the database specifies a maximum document size and the document
+        exceeds it, put will fail and raise a DocumentTooBig exception.
+
+        :param doc: A Document with new content.
+        :return: new_doc_rev - The new revision identifier for the document.
+            The Document object will also be updated.
+
+        :raise InvalidDocId: Raised if the document's id is invalid.
+        :raise DocumentTooBig: Raised if the document size is too big.
+        :raise ConflictedDoc: Raised if the document has conflicts.
+        """
+        if doc.doc_id is None:
+            raise InvalidDocId()
+        self._check_doc_id(doc.doc_id)
+        self._check_doc_size(doc)
+        old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
+        if old_doc and old_doc.has_conflicts:
+            raise ConflictedDoc()
+        if old_doc and doc.rev is None and old_doc.is_tombstone():
+            new_rev = self._allocate_doc_rev(old_doc.rev)
+        else:
+            if old_doc is not None:
+                    if old_doc.rev != doc.rev:
+                        raise RevisionConflict()
+            else:
+                if doc.rev is not None:
+                    raise RevisionConflict()
+            new_rev = self._allocate_doc_rev(doc.rev)
+        doc.rev = new_rev
+        self._put_doc(old_doc, doc)
+        return new_rev
+
+    def whats_changed(self, old_generation=0):
+        """
+        Return a list of documents that have changed since old_generation.
+
+        :param old_generation: The generation of the database in the old
+                               state.
+        :type old_generation: int
+
+        :return: (generation, trans_id, [(doc_id, generation, trans_id),...])
+                 The current generation of the database, its associated
+                 transaction id, and a list of of changed documents since
+                 old_generation, represented by tuples with for each document
+                 its doc_id and the generation and transaction id corresponding
+                 to the last intervening change and sorted by generation (old
+                 changes first)
+        :rtype: (int, str, [(str, int, str)])
+        """
+        return self._database.whats_changed(old_generation)
+
+    def delete_doc(self, doc):
+        """
+        Mark a document as deleted.
+
+        Will abort if the current revision doesn't match doc.rev.
+        This will also set doc.content to None.
+
+        :param doc: The document to mark as deleted.
+        :type doc: CouchDocument.
+
+        :raise DocumentDoesNotExist: Raised if the document does not
+                                            exist.
+        :raise RevisionConflict: Raised if the revisions do not match.
+        :raise DocumentAlreadyDeleted: Raised if the document is
+                                              already deleted.
+        :raise ConflictedDoc: Raised if the doc has conflicts.
+        """
+        old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
+        if old_doc is None:
+            raise DocumentDoesNotExist
+        if old_doc.rev != doc.rev:
+            raise RevisionConflict()
+        if old_doc.is_tombstone():
+            raise DocumentAlreadyDeleted
+        if old_doc.has_conflicts:
+            raise ConflictedDoc()
+        new_rev = self._allocate_doc_rev(doc.rev)
+        doc.rev = new_rev
+        doc.make_tombstone()
+        self._put_doc(old_doc, doc)
+        return new_rev
+
+    def get_doc_conflicts(self, doc_id, couch_rev=None):
+        """
+        Get the conflicted versions of a document.
+
+        If the C{couch_rev} parameter is not None, conflicts for a specific
+        document's couch revision are returned.
+
+        :param couch_rev: The couch document revision.
+        :type couch_rev: str
+
+        :return: A list of conflicted versions of the document.
+        :rtype: list
+        """
+        return self._database.get_doc_conflicts(doc_id, couch_rev)
+
+    def _get_replica_gen_and_trans_id(self, other_replica_uid):
+        """
+        Return the last known generation and transaction id for the other db
+        replica.
+
+        When you do a synchronization with another replica, the Database keeps
+        track of what generation the other database replica was at, and what
+        the associated transaction id was.  This is used to determine what data
+        needs to be sent, and if two databases are claiming to be the same
+        replica.
+
+        :param other_replica_uid: The identifier for the other replica.
+        :type other_replica_uid: str
+
+        :return: A tuple containing the generation and transaction id we
+                 encountered during synchronization. If we've never
+                 synchronized with the replica, this is (0, '').
+        :rtype: (int, str)
+        """
+        return self._database._get_replica_gen_and_trans_id(other_replica_uid)
+
+    def _set_replica_gen_and_trans_id(self, other_replica_uid,
+                                      other_generation, other_transaction_id,
+                                      number_of_docs=None, doc_idx=None,
+                                      sync_id=None):
+        """
+        Set the last-known generation and transaction id for the other
+        database replica.
+
+        We have just performed some synchronization, and we want to track what
+        generation the other replica was at. See also
+        _get_replica_gen_and_trans_id.
+
+        :param other_replica_uid: The U1DB identifier for the other replica.
+        :type other_replica_uid: str
+        :param other_generation: The generation number for the other replica.
+        :type other_generation: int
+        :param other_transaction_id: The transaction id associated with the
+            generation.
+        :type other_transaction_id: str
+        :param number_of_docs: The total amount of documents sent on this sync
+                               session.
+        :type number_of_docs: int
+        :param doc_idx: The index of the current document being sent.
+        :type doc_idx: int
+        :param sync_id: The id of the current sync session.
+        :type sync_id: str
+        """
+        if other_replica_uid is not None and other_generation is not None:
+            self._do_set_replica_gen_and_trans_id(
+                other_replica_uid, other_generation, other_transaction_id,
+                number_of_docs=number_of_docs, doc_idx=doc_idx,
+                sync_id=sync_id)
+
+    def _do_set_replica_gen_and_trans_id(
+            self, other_replica_uid, other_generation, other_transaction_id,
+            number_of_docs=None, doc_idx=None, sync_id=None):
+        """
+        Set the last-known generation and transaction id for the other
+        database replica.
+
+        We have just performed some synchronization, and we want to track what
+        generation the other replica was at. See also
+        _get_replica_gen_and_trans_id.
+
+        :param other_replica_uid: The U1DB identifier for the other replica.
+        :type other_replica_uid: str
+        :param other_generation: The generation number for the other replica.
+        :type other_generation: int
+        :param other_transaction_id: The transaction id associated with the
+                                     generation.
+        :type other_transaction_id: str
+        :param number_of_docs: The total amount of documents sent on this sync
+                               session.
+        :type number_of_docs: int
+        :param doc_idx: The index of the current document being sent.
+        :type doc_idx: int
+        :param sync_id: The id of the current sync session.
+        :type sync_id: str
+        """
+        self.cache[other_replica_uid] = (other_generation,
+                                         other_transaction_id)
+        self._database._do_set_replica_gen_and_trans_id(other_replica_uid,
+                                                        other_generation,
+                                                        other_transaction_id)
+
+    def _force_doc_sync_conflict(self, doc):
+        """
+        Add a conflict and force a document put.
+
+        :param doc: The document to be put.
+        :type doc: CouchDocument
+        """
+        my_doc = self._get_doc(doc.doc_id)
+        self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev))
+        doc.add_conflict(self._factory(doc.doc_id, my_doc.rev,
+                                       my_doc.get_json()))
+        doc.has_conflicts = True
+        self._put_doc(my_doc, doc)
+
+    def resolve_doc(self, doc, conflicted_doc_revs):
+        """
+        Mark a document as no longer conflicted.
+
+        We take the list of revisions that the client knows about that it is
+        superseding. This may be a different list from the actual current
+        conflicts, in which case only those are removed as conflicted.  This
+        may fail if the conflict list is significantly different from the
+        supplied information. (sync could have happened in the background from
+        the time you GET_DOC_CONFLICTS until the point where you RESOLVE)
+
+        :param doc: A Document with the new content to be inserted.
+        :type doc: CouchDocument
+        :param conflicted_doc_revs: A list of revisions that the new content
+                                    supersedes.
+        :type conflicted_doc_revs: [str]
+
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        cur_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
+        new_rev = self._ensure_maximal_rev(cur_doc.rev,
+                                           conflicted_doc_revs)
+        superseded_revs = set(conflicted_doc_revs)
+        doc.rev = new_rev
+        # this backend stores conflicts as properties of the documents, so we
+        # have to copy these conflicts over to the document being updated.
+        if cur_doc.rev in superseded_revs:
+            # the newer doc version will supersede the one in the database, so
+            # we copy conflicts before updating the backend.
+            doc.set_conflicts(cur_doc.get_conflicts())  # copy conflicts over.
+            doc.delete_conflicts(superseded_revs)
+            self._put_doc(cur_doc, doc)
+        else:
+            # the newer doc version does not supersede the one in the
+            # database, so we will add a conflict to the database and copy
+            # those over to the document the user has in her hands.
+            cur_doc.add_conflict(doc)
+            cur_doc.delete_conflicts(superseded_revs)
+            self._put_doc(cur_doc, cur_doc)  # just update conflicts
+            # backend has been updated with current conflicts, now copy them
+            # to the current document.
+            doc.set_conflicts(cur_doc.get_conflicts())
+
+    def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen,
+                          replica_trans_id='', number_of_docs=None,
+                          doc_idx=None, sync_id=None):
+        """
+        Insert/update document into the database with a given revision.
+
+        This api is used during synchronization operations.
+
+        If a document would conflict and save_conflict is set to True, the
+        content will be selected as the 'current' content for doc.doc_id,
+        even though doc.rev doesn't supersede the currently stored revision.
+        The currently stored document will be added to the list of conflict
+        alternatives for the given doc_id.
+
+        This forces the new content to be 'current' so that we get convergence
+        after synchronizing, even if people don't resolve conflicts. Users can
+        then notice that their content is out of date, update it, and
+        synchronize again. (The alternative is that users could synchronize and
+        think the data has propagated, but their local copy looks fine, and the
+        remote copy is never updated again.)
+
+        :param doc: A document object
+        :type doc: CouchDocument
+        :param save_conflict: If this document is a conflict, do you want to
+                              save it as a conflict, or just ignore it.
+        :type save_conflict: bool
+        :param replica_uid: A unique replica identifier.
+        :type replica_uid: str
+        :param replica_gen: The generation of the replica corresponding to the
+                            this document. The replica arguments are optional,
+                            but are used during synchronization.
+        :type replica_gen: int
+        :param replica_trans_id: The transaction_id associated with the
+                                 generation.
+        :type replica_trans_id: str
+        :param number_of_docs: The total amount of documents sent on this sync
+                               session.
+        :type number_of_docs: int
+        :param doc_idx: The index of the current document being sent.
+        :type doc_idx: int
+        :param sync_id: The id of the current sync session.
+        :type sync_id: str
+
+        :return: (state, at_gen) -  If we don't have doc_id already, or if
+                 doc_rev supersedes the existing document revision, then the
+                 content will be inserted, and state is 'inserted'.  If
+                 doc_rev is less than or equal to the existing revision, then
+                 the put is ignored and state is respecitvely 'superseded' or
+                 'converged'.  If doc_rev is not strictly superseded or
+                 supersedes, then state is 'conflicted'. The document will not
+                 be inserted if save_conflict is False.  For 'inserted' or
+                 'converged', at_gen is the insertion/current generation.
+        :rtype: (str, int)
+        """
+        if not isinstance(doc, CouchDocument):
+            doc = self._factory(doc.doc_id, doc.rev, doc.get_json())
+        my_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
+        if my_doc:
+            doc.set_conflicts(my_doc.get_conflicts())
+        return CommonBackend._put_doc_if_newer(self, doc, save_conflict,
+                                               replica_uid, replica_gen,
+                                               replica_trans_id)
+
+    def _put_and_update_indexes(self, cur_doc, doc):
+        self._put_doc(cur_doc, doc)
+
+    def get_docs(self, doc_ids, check_for_conflicts=True,
+                 include_deleted=False):
+        """
+        Get the JSON content for many documents.
+
+        :param doc_ids: A list of document identifiers or None for all.
+        :type doc_ids: list
+        :param check_for_conflicts: If set to False, then the conflict check
+                                    will be skipped, and 'None' will be
+                                    returned instead of True/False.
+        :type check_for_conflicts: bool
+        :param include_deleted: If set to True, deleted documents will be
+                                returned with empty content. Otherwise deleted
+                                documents will not be included in the results.
+        :return: iterable giving the Document object for each document id
+                 in matching doc_ids order.
+        :rtype: iterable
+        """
+        return self._database.get_docs(doc_ids, check_for_conflicts,
+                                       include_deleted)
+
+    def _prune_conflicts(self, doc, doc_vcr):
+        """
+        Prune conflicts that are older then the current document's revision, or
+        whose content match to the current document's content.
+        Originally in u1db.CommonBackend
+
+        :param doc: The document to have conflicts pruned.
+        :type doc: CouchDocument
+        :param doc_vcr: A vector clock representing the current document's
+                        revision.
+        :type doc_vcr: u1db.vectorclock.VectorClock
+        """
+        if doc.has_conflicts:
+            autoresolved = False
+            c_revs_to_prune = []
+            for c_doc in doc._conflicts:
+                c_vcr = vectorclock.VectorClockRev(c_doc.rev)
+                if doc_vcr.is_newer(c_vcr):
+                    c_revs_to_prune.append(c_doc.rev)
+                elif doc.same_content_as(c_doc):
+                    c_revs_to_prune.append(c_doc.rev)
+                    doc_vcr.maximize(c_vcr)
+                    autoresolved = True
+            if autoresolved:
+                doc_vcr.increment(self._replica_uid)
+                doc.rev = doc_vcr.as_str()
+            doc.delete_conflicts(c_revs_to_prune)
+
+
+class SoledadSyncTarget(CommonSyncTarget):
+
+    """
+    Functionality for using a SoledadBackend as a synchronization target.
+    """
+
+    def get_sync_info(self, source_replica_uid):
+        source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id(
+            source_replica_uid)
+        my_gen, my_trans_id = self._db._get_generation_info()
+        return (
+            self._db._replica_uid, my_gen, my_trans_id, source_gen,
+            source_trans_id)
+
+    def record_sync_info(self, source_replica_uid, source_replica_generation,
+                         source_replica_transaction_id):
+        if self._trace_hook:
+            self._trace_hook('record_sync_info')
+        self._db._set_replica_gen_and_trans_id(
+            source_replica_uid, source_replica_generation,
+            source_replica_transaction_id)
diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py
deleted file mode 100644
index d4f67696..00000000
--- a/common/src/leap/soledad/common/couch.py
+++ /dev/null
@@ -1,1514 +0,0 @@
-# -*- coding: utf-8 -*-
-# couch.py
-# Copyright (C) 2013 LEAP
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-"""A U1DB backend that uses CouchDB as its persistence layer."""
-
-
-import json
-import re
-import uuid
-import logging
-import binascii
-import time
-
-
-from StringIO import StringIO
-from urlparse import urljoin
-from contextlib import contextmanager
-from multiprocessing.pool import ThreadPool
-
-
-from couchdb.client import Server, Database
-from couchdb.multipart import MultipartWriter
-from couchdb.http import (
-    ResourceConflict,
-    ResourceNotFound,
-    ServerError,
-    Session,
-    urljoin as couch_urljoin,
-    Resource,
-)
-from u1db import vectorclock
-from u1db.errors import (
-    DatabaseDoesNotExist,
-    InvalidGeneration,
-    RevisionConflict,
-    InvalidDocId,
-    ConflictedDoc,
-    DocumentDoesNotExist,
-    DocumentAlreadyDeleted,
-    Unauthorized,
-)
-from u1db.backends import CommonBackend, CommonSyncTarget
-from u1db.remote import http_app
-from u1db.remote.server_state import ServerState
-
-
-from leap.soledad.common import ddocs
-from leap.soledad.common.errors import raise_server_error
-from leap.soledad.common.errors import raise_missing_design_doc_error
-from leap.soledad.common.errors import InvalidURLError
-from leap.soledad.common.command import exec_validated_cmd
-from leap.soledad.common.document import CouchDocument
-
-
-logger = logging.getLogger(__name__)
-
-
-COUCH_TIMEOUT = 120  # timeout for transfers between Soledad server and Couch
-
-
-def list_users_dbs(couch_url):
-    """
-    Retrieves a list with all databases that starts with 'user-' on CouchDB.
-    Those databases belongs to users. So, the list will contain all the
-    database names in the form of 'user-{uuid4}'.
-
-    :param couch_url: The couch url with needed credentials
-    :type couch_url: str
-
-    :return: The list of all database names from users.
-    :rtype: [str]
-    """
-    with couch_server(couch_url) as server:
-        users = [dbname for dbname in server if dbname.startswith('user-')]
-    return users
-
-# monkey-patch the u1db http app to use CouchDocument
-http_app.Document = CouchDocument
-
-
-@contextmanager
-def couch_server(url):
-    """
-    Provide a connection to a couch server and cleanup after use.
-
-    For database creation and deletion we use an ephemeral connection to the
-    couch server. That connection has to be properly closed, so we provide it
-    as a context manager.
-
-    :param url: The URL of the Couch server.
-    :type url: str
-    """
-    session = Session(timeout=COUCH_TIMEOUT)
-    server = Server(url=url, full_commit=False, session=session)
-    yield server
-
-
-THREAD_POOL = ThreadPool(20)
-
-
-class SoledadBackend(CommonBackend):
-
-    """
-    A U1DB implementation that uses CouchDB as its persistence layer.
-    """
-
-    @classmethod
-    def open_database(cls, url, create, replica_uid=None, ensure_ddocs=False,
-                      database_security=None):
-        """
-        Open a U1DB database using CouchDB as backend.
-
-        :param url: the url of the database replica
-        :type url: str
-        :param create: should the replica be created if it does not exist?
-        :type create: bool
-        :param replica_uid: an optional unique replica identifier
-        :type replica_uid: str
-        :param ensure_ddocs: Ensure that the design docs exist on server.
-        :type ensure_ddocs: bool
-
-        :return: the database instance
-        :rtype: SoledadBackend
-        """
-        db = CouchDatabase.open_database(url, create, ensure_ddocs)
-        return cls(
-            db, replica_uid=replica_uid)
-
-    def __init__(self, database, replica_uid=None):
-        """
-        Create a new Couch data container.
-
-        :param url: the url of the couch database
-        :type url: str
-        :param dbname: the database name
-        :type dbname: str
-        :param replica_uid: an optional unique replica identifier
-        :type replica_uid: str
-        :param ensure_ddocs: Ensure that the design docs exist on server.
-        :type ensure_ddocs: bool
-        """
-        # save params
-        self._factory = CouchDocument
-        self._real_replica_uid = None
-        # configure couch
-        self._cache = None
-        self._dbname = database._dbname
-        self._database = database
-        if replica_uid is not None:
-            self._set_replica_uid(replica_uid)
-
-    @property
-    def cache(self):
-        if self._cache is not None:
-            return self._cache
-        else:
-            return {}
-
-    def init_caching(self, cache):
-        """
-        Start using cache by setting internal _cache attribute.
-
-        :param cache: the cache instance, anything that behaves like a dict
-        :type cache: dict
-        """
-        self._cache = cache
-
-    def get_sync_target(self):
-        """
-        Return a SyncTarget object, for another u1db to synchronize with.
-
-        :return: The sync target.
-        :rtype: CouchSyncTarget
-        """
-        return CouchSyncTarget(self)
-
-    def delete_database(self):
-        """
-        Delete a U1DB CouchDB database.
-        """
-        self._database.delete_database()
-
-    def close(self):
-        """
-        Release any resources associated with this database.
-
-        :return: True if db was succesfully closed.
-        :rtype: bool
-        """
-        self._database.close()
-        return True
-
-    def __del__(self):
-        """
-        Close the database upon garbage collection.
-        """
-        self.close()
-
-    def _set_replica_uid(self, replica_uid):
-        """
-        Force the replica uid to be set.
-
-        :param replica_uid: The new replica uid.
-        :type replica_uid: str
-        """
-        self._database.set_replica_uid(replica_uid)
-        self._real_replica_uid = replica_uid
-        self.cache['replica_uid'] = self._real_replica_uid
-
-    def _get_replica_uid(self):
-        """
-        Get the replica uid.
-
-        :return: The replica uid.
-        :rtype: str
-        """
-        if self._real_replica_uid is not None:
-            self.cache['replica_uid'] = self._real_replica_uid
-            return self._real_replica_uid
-        if 'replica_uid' in self.cache:
-            return self.cache['replica_uid']
-        self._real_replica_uid = self._database.get_replica_uid()
-        self._set_replica_uid(self._real_replica_uid)
-        return self._real_replica_uid
-
-    _replica_uid = property(_get_replica_uid, _set_replica_uid)
-
-    replica_uid = property(_get_replica_uid)
-
-    def _get_generation(self):
-        """
-        Return the current generation.
-
-        :return: The current generation.
-        :rtype: int
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
-        """
-        return self._get_generation_info()[0]
-
-    def _get_generation_info(self):
-        """
-        Return the current generation.
-
-        :return: A tuple containing the current generation and transaction id.
-        :rtype: (int, str)
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
-        """
-        if self.replica_uid + '_gen' in self.cache:
-            response = self.cache[self.replica_uid + '_gen']
-            return response
-        cur_gen, newest_trans_id = self._database._get_generation_info()
-        self.cache[self.replica_uid + '_gen'] = (cur_gen, newest_trans_id)
-        return (cur_gen, newest_trans_id)
-
-    def _get_trans_id_for_gen(self, generation):
-        """
-        Get the transaction id corresponding to a particular generation.
-
-        :param generation: The generation for which to get the transaction id.
-        :type generation: int
-
-        :return: The transaction id for C{generation}.
-        :rtype: str
-
-        """
-        return self._database._get_trans_id_for_gen(generation)
-
-    def _get_transaction_log(self):
-        """
-        This is only for the test suite, it is not part of the api.
-
-        :return: The complete transaction log.
-        :rtype: [(str, str)]
-
-        """
-        return self._database._get_transaction_log()
-
-    def _get_doc(self, doc_id, check_for_conflicts=False):
-        """
-        Extract the document from storage.
-
-        This can return None if the document doesn't exist.
-
-        :param doc_id: The unique document identifier
-        :type doc_id: str
-        :param check_for_conflicts: If set to False, then the conflict check
-                                    will be skipped.
-        :type check_for_conflicts: bool
-
-        :return: The document.
-        :rtype: CouchDocument
-        """
-        return self._database._get_doc(doc_id, check_for_conflicts)
-
-    def get_doc(self, doc_id, include_deleted=False):
-        """
-        Get the JSON string for the given document.
-
-        :param doc_id: The unique document identifier
-        :type doc_id: str
-        :param include_deleted: If set to True, deleted documents will be
-            returned with empty content. Otherwise asking for a deleted
-            document will return None.
-        :type include_deleted: bool
-
-        :return: A document object.
-        :rtype: CouchDocument.
-        """
-        doc = self._get_doc(doc_id, check_for_conflicts=True)
-        if doc is None:
-            return None
-        if doc.is_tombstone() and not include_deleted:
-            return None
-        return doc
-
-    def get_all_docs(self, include_deleted=False):
-        """
-        Get the JSON content for all documents in the database.
-
-        :param include_deleted: If set to True, deleted documents will be
-                                returned with empty content. Otherwise deleted
-                                documents will not be included in the results.
-        :type include_deleted: bool
-
-        :return: (generation, [CouchDocument])
-            The current generation of the database, followed by a list of all
-            the documents in the database.
-        :rtype: (int, [CouchDocument])
-        """
-        return self._database.get_all_docs(include_deleted)
-
-    def _put_doc(self, old_doc, doc):
-        """
-        Put the document in the Couch backend database.
-
-        Note that C{old_doc} must have been fetched with the parameter
-        C{check_for_conflicts} equal to True, so we can properly update the
-        new document using the conflict information from the old one.
-
-        :param old_doc: The old document version.
-        :type old_doc: CouchDocument
-        :param doc: The document to be put.
-        :type doc: CouchDocument
-        """
-        last_transaction =\
-            self._database.save_document(old_doc, doc,
-                                         self._allocate_transaction_id())
-        if self.replica_uid + '_gen' in self.cache:
-            gen, trans = self.cache[self.replica_uid + '_gen']
-            gen += 1
-            trans = last_transaction
-            self.cache[self.replica_uid + '_gen'] = (gen, trans)
-
-    def put_doc(self, doc):
-        """
-        Update a document.
-
-        If the document currently has conflicts, put will fail.
-        If the database specifies a maximum document size and the document
-        exceeds it, put will fail and raise a DocumentTooBig exception.
-
-        :param doc: A Document with new content.
-        :return: new_doc_rev - The new revision identifier for the document.
-            The Document object will also be updated.
-
-        :raise InvalidDocId: Raised if the document's id is invalid.
-        :raise DocumentTooBig: Raised if the document size is too big.
-        :raise ConflictedDoc: Raised if the document has conflicts.
-        """
-        if doc.doc_id is None:
-            raise InvalidDocId()
-        self._check_doc_id(doc.doc_id)
-        self._check_doc_size(doc)
-        old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
-        if old_doc and old_doc.has_conflicts:
-            raise ConflictedDoc()
-        if old_doc and doc.rev is None and old_doc.is_tombstone():
-            new_rev = self._allocate_doc_rev(old_doc.rev)
-        else:
-            if old_doc is not None:
-                    if old_doc.rev != doc.rev:
-                        raise RevisionConflict()
-            else:
-                if doc.rev is not None:
-                    raise RevisionConflict()
-            new_rev = self._allocate_doc_rev(doc.rev)
-        doc.rev = new_rev
-        self._put_doc(old_doc, doc)
-        return new_rev
-
-    def whats_changed(self, old_generation=0):
-        """
-        Return a list of documents that have changed since old_generation.
-
-        :param old_generation: The generation of the database in the old
-                               state.
-        :type old_generation: int
-
-        :return: (generation, trans_id, [(doc_id, generation, trans_id),...])
-                 The current generation of the database, its associated
-                 transaction id, and a list of of changed documents since
-                 old_generation, represented by tuples with for each document
-                 its doc_id and the generation and transaction id corresponding
-                 to the last intervening change and sorted by generation (old
-                 changes first)
-        :rtype: (int, str, [(str, int, str)])
-        """
-        return self._database.whats_changed(old_generation)
-
-    def delete_doc(self, doc):
-        """
-        Mark a document as deleted.
-
-        Will abort if the current revision doesn't match doc.rev.
-        This will also set doc.content to None.
-
-        :param doc: The document to mark as deleted.
-        :type doc: CouchDocument.
-
-        :raise DocumentDoesNotExist: Raised if the document does not
-                                            exist.
-        :raise RevisionConflict: Raised if the revisions do not match.
-        :raise DocumentAlreadyDeleted: Raised if the document is
-                                              already deleted.
-        :raise ConflictedDoc: Raised if the doc has conflicts.
-        """
-        old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
-        if old_doc is None:
-            raise DocumentDoesNotExist
-        if old_doc.rev != doc.rev:
-            raise RevisionConflict()
-        if old_doc.is_tombstone():
-            raise DocumentAlreadyDeleted
-        if old_doc.has_conflicts:
-            raise ConflictedDoc()
-        new_rev = self._allocate_doc_rev(doc.rev)
-        doc.rev = new_rev
-        doc.make_tombstone()
-        self._put_doc(old_doc, doc)
-        return new_rev
-
-    def get_doc_conflicts(self, doc_id, couch_rev=None):
-        """
-        Get the conflicted versions of a document.
-
-        If the C{couch_rev} parameter is not None, conflicts for a specific
-        document's couch revision are returned.
-
-        :param couch_rev: The couch document revision.
-        :type couch_rev: str
-
-        :return: A list of conflicted versions of the document.
-        :rtype: list
-        """
-        return self._database.get_doc_conflicts(doc_id, couch_rev)
-
-    def _get_replica_gen_and_trans_id(self, other_replica_uid):
-        """
-        Return the last known generation and transaction id for the other db
-        replica.
-
-        When you do a synchronization with another replica, the Database keeps
-        track of what generation the other database replica was at, and what
-        the associated transaction id was.  This is used to determine what data
-        needs to be sent, and if two databases are claiming to be the same
-        replica.
-
-        :param other_replica_uid: The identifier for the other replica.
-        :type other_replica_uid: str
-
-        :return: A tuple containing the generation and transaction id we
-                 encountered during synchronization. If we've never
-                 synchronized with the replica, this is (0, '').
-        :rtype: (int, str)
-        """
-        return self._database._get_replica_gen_and_trans_id(other_replica_uid)
-
-    def _set_replica_gen_and_trans_id(self, other_replica_uid,
-                                      other_generation, other_transaction_id,
-                                      number_of_docs=None, doc_idx=None,
-                                      sync_id=None):
-        """
-        Set the last-known generation and transaction id for the other
-        database replica.
-
-        We have just performed some synchronization, and we want to track what
-        generation the other replica was at. See also
-        _get_replica_gen_and_trans_id.
-
-        :param other_replica_uid: The U1DB identifier for the other replica.
-        :type other_replica_uid: str
-        :param other_generation: The generation number for the other replica.
-        :type other_generation: int
-        :param other_transaction_id: The transaction id associated with the
-            generation.
-        :type other_transaction_id: str
-        :param number_of_docs: The total amount of documents sent on this sync
-                               session.
-        :type number_of_docs: int
-        :param doc_idx: The index of the current document being sent.
-        :type doc_idx: int
-        :param sync_id: The id of the current sync session.
-        :type sync_id: str
-        """
-        if other_replica_uid is not None and other_generation is not None:
-            self._do_set_replica_gen_and_trans_id(
-                other_replica_uid, other_generation, other_transaction_id,
-                number_of_docs=number_of_docs, doc_idx=doc_idx,
-                sync_id=sync_id)
-
-    def _do_set_replica_gen_and_trans_id(
-            self, other_replica_uid, other_generation, other_transaction_id,
-            number_of_docs=None, doc_idx=None, sync_id=None):
-        """
-        Set the last-known generation and transaction id for the other
-        database replica.
-
-        We have just performed some synchronization, and we want to track what
-        generation the other replica was at. See also
-        _get_replica_gen_and_trans_id.
-
-        :param other_replica_uid: The U1DB identifier for the other replica.
-        :type other_replica_uid: str
-        :param other_generation: The generation number for the other replica.
-        :type other_generation: int
-        :param other_transaction_id: The transaction id associated with the
-                                     generation.
-        :type other_transaction_id: str
-        :param number_of_docs: The total amount of documents sent on this sync
-                               session.
-        :type number_of_docs: int
-        :param doc_idx: The index of the current document being sent.
-        :type doc_idx: int
-        :param sync_id: The id of the current sync session.
-        :type sync_id: str
-        """
-        self.cache[other_replica_uid] = (other_generation,
-                                         other_transaction_id)
-        self._database._do_set_replica_gen_and_trans_id(other_replica_uid,
-                                                        other_generation,
-                                                        other_transaction_id)
-
-    def _force_doc_sync_conflict(self, doc):
-        """
-        Add a conflict and force a document put.
-
-        :param doc: The document to be put.
-        :type doc: CouchDocument
-        """
-        my_doc = self._get_doc(doc.doc_id)
-        self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev))
-        doc.add_conflict(self._factory(doc.doc_id, my_doc.rev,
-                                       my_doc.get_json()))
-        doc.has_conflicts = True
-        self._put_doc(my_doc, doc)
-
-    def resolve_doc(self, doc, conflicted_doc_revs):
-        """
-        Mark a document as no longer conflicted.
-
-        We take the list of revisions that the client knows about that it is
-        superseding. This may be a different list from the actual current
-        conflicts, in which case only those are removed as conflicted.  This
-        may fail if the conflict list is significantly different from the
-        supplied information. (sync could have happened in the background from
-        the time you GET_DOC_CONFLICTS until the point where you RESOLVE)
-
-        :param doc: A Document with the new content to be inserted.
-        :type doc: CouchDocument
-        :param conflicted_doc_revs: A list of revisions that the new content
-                                    supersedes.
-        :type conflicted_doc_revs: [str]
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
-        """
-        cur_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
-        new_rev = self._ensure_maximal_rev(cur_doc.rev,
-                                           conflicted_doc_revs)
-        superseded_revs = set(conflicted_doc_revs)
-        doc.rev = new_rev
-        # this backend stores conflicts as properties of the documents, so we
-        # have to copy these conflicts over to the document being updated.
-        if cur_doc.rev in superseded_revs:
-            # the newer doc version will supersede the one in the database, so
-            # we copy conflicts before updating the backend.
-            doc.set_conflicts(cur_doc.get_conflicts())  # copy conflicts over.
-            doc.delete_conflicts(superseded_revs)
-            self._put_doc(cur_doc, doc)
-        else:
-            # the newer doc version does not supersede the one in the
-            # database, so we will add a conflict to the database and copy
-            # those over to the document the user has in her hands.
-            cur_doc.add_conflict(doc)
-            cur_doc.delete_conflicts(superseded_revs)
-            self._put_doc(cur_doc, cur_doc)  # just update conflicts
-            # backend has been updated with current conflicts, now copy them
-            # to the current document.
-            doc.set_conflicts(cur_doc.get_conflicts())
-
-    def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen,
-                          replica_trans_id='', number_of_docs=None,
-                          doc_idx=None, sync_id=None):
-        """
-        Insert/update document into the database with a given revision.
-
-        This api is used during synchronization operations.
-
-        If a document would conflict and save_conflict is set to True, the
-        content will be selected as the 'current' content for doc.doc_id,
-        even though doc.rev doesn't supersede the currently stored revision.
-        The currently stored document will be added to the list of conflict
-        alternatives for the given doc_id.
-
-        This forces the new content to be 'current' so that we get convergence
-        after synchronizing, even if people don't resolve conflicts. Users can
-        then notice that their content is out of date, update it, and
-        synchronize again. (The alternative is that users could synchronize and
-        think the data has propagated, but their local copy looks fine, and the
-        remote copy is never updated again.)
-
-        :param doc: A document object
-        :type doc: CouchDocument
-        :param save_conflict: If this document is a conflict, do you want to
-                              save it as a conflict, or just ignore it.
-        :type save_conflict: bool
-        :param replica_uid: A unique replica identifier.
-        :type replica_uid: str
-        :param replica_gen: The generation of the replica corresponding to the
-                            this document. The replica arguments are optional,
-                            but are used during synchronization.
-        :type replica_gen: int
-        :param replica_trans_id: The transaction_id associated with the
-                                 generation.
-        :type replica_trans_id: str
-        :param number_of_docs: The total amount of documents sent on this sync
-                               session.
-        :type number_of_docs: int
-        :param doc_idx: The index of the current document being sent.
-        :type doc_idx: int
-        :param sync_id: The id of the current sync session.
-        :type sync_id: str
-
-        :return: (state, at_gen) -  If we don't have doc_id already, or if
-                 doc_rev supersedes the existing document revision, then the
-                 content will be inserted, and state is 'inserted'.  If
-                 doc_rev is less than or equal to the existing revision, then
-                 the put is ignored and state is respecitvely 'superseded' or
-                 'converged'.  If doc_rev is not strictly superseded or
-                 supersedes, then state is 'conflicted'. The document will not
-                 be inserted if save_conflict is False.  For 'inserted' or
-                 'converged', at_gen is the insertion/current generation.
-        :rtype: (str, int)
-        """
-        if not isinstance(doc, CouchDocument):
-            doc = self._factory(doc.doc_id, doc.rev, doc.get_json())
-        my_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
-        if my_doc:
-            doc.set_conflicts(my_doc.get_conflicts())
-        return CommonBackend._put_doc_if_newer(self, doc, save_conflict,
-                                               replica_uid, replica_gen,
-                                               replica_trans_id)
-
-    def _put_and_update_indexes(self, cur_doc, doc):
-        self._put_doc(cur_doc, doc)
-
-    def get_docs(self, doc_ids, check_for_conflicts=True,
-                 include_deleted=False):
-        """
-        Get the JSON content for many documents.
-
-        :param doc_ids: A list of document identifiers or None for all.
-        :type doc_ids: list
-        :param check_for_conflicts: If set to False, then the conflict check
-                                    will be skipped, and 'None' will be
-                                    returned instead of True/False.
-        :type check_for_conflicts: bool
-        :param include_deleted: If set to True, deleted documents will be
-                                returned with empty content. Otherwise deleted
-                                documents will not be included in the results.
-        :return: iterable giving the Document object for each document id
-                 in matching doc_ids order.
-        :rtype: iterable
-        """
-        return self._database.get_docs(doc_ids, check_for_conflicts,
-                                       include_deleted)
-
-    def _prune_conflicts(self, doc, doc_vcr):
-        """
-        Prune conflicts that are older then the current document's revision, or
-        whose content match to the current document's content.
-        Originally in u1db.CommonBackend
-
-        :param doc: The document to have conflicts pruned.
-        :type doc: CouchDocument
-        :param doc_vcr: A vector clock representing the current document's
-                        revision.
-        :type doc_vcr: u1db.vectorclock.VectorClock
-        """
-        if doc.has_conflicts:
-            autoresolved = False
-            c_revs_to_prune = []
-            for c_doc in doc._conflicts:
-                c_vcr = vectorclock.VectorClockRev(c_doc.rev)
-                if doc_vcr.is_newer(c_vcr):
-                    c_revs_to_prune.append(c_doc.rev)
-                elif doc.same_content_as(c_doc):
-                    c_revs_to_prune.append(c_doc.rev)
-                    doc_vcr.maximize(c_vcr)
-                    autoresolved = True
-            if autoresolved:
-                doc_vcr.increment(self._replica_uid)
-                doc.rev = doc_vcr.as_str()
-            doc.delete_conflicts(c_revs_to_prune)
-
-
-class CouchDatabase(object):
-    """
-    Holds CouchDB related code.
-    This class gives methods to encapsulate database operations and hide
-    CouchDB details from backend code.
-    """
-
-    @classmethod
-    def open_database(cls, url, create, ensure_ddocs=False):
-        """
-        Open a U1DB database using CouchDB as backend.
-
-        :param url: the url of the database replica
-        :type url: str
-        :param create: should the replica be created if it does not exist?
-        :type create: bool
-        :param replica_uid: an optional unique replica identifier
-        :type replica_uid: str
-        :param ensure_ddocs: Ensure that the design docs exist on server.
-        :type ensure_ddocs: bool
-
-        :return: the database instance
-        :rtype: SoledadBackend
-        """
-        # get database from url
-        m = re.match('(^https?://[^/]+)/(.+)$', url)
-        if not m:
-            raise InvalidURLError
-        url = m.group(1)
-        dbname = m.group(2)
-        with couch_server(url) as server:
-            try:
-                server[dbname]
-            except ResourceNotFound:
-                if not create:
-                    raise DatabaseDoesNotExist()
-                server.create(dbname)
-        return cls(
-            url, dbname, ensure_ddocs=ensure_ddocs)
-
-    def __init__(self, url, dbname, ensure_ddocs=True,
-                 database_security=None):
-        self._session = Session(timeout=COUCH_TIMEOUT)
-        self._url = url
-        self._dbname = dbname
-        self._database = Database(
-            urljoin(url, dbname),
-            self._session)
-        self._database.info()
-        if ensure_ddocs:
-            self.ensure_ddocs_on_db()
-            self.ensure_security_ddoc(database_security)
-
-    def ensure_ddocs_on_db(self):
-        """
-        Ensure that the design documents used by the backend exist on the
-        couch database.
-        """
-        for ddoc_name in ['docs', 'syncs', 'transactions']:
-            try:
-                self._database.resource('_design',
-                                        ddoc_name, '_info').get_json()
-            except ResourceNotFound:
-                ddoc = json.loads(
-                    binascii.a2b_base64(
-                        getattr(ddocs, ddoc_name)))
-                self._database.save(ddoc)
-
-    def ensure_security_ddoc(self, security_config=None):
-        """
-        Make sure that only soledad user is able to access this database as
-        an unprivileged member, meaning that administration access will
-        be forbidden even inside an user database.
-        The goal is to make sure that only the lowest access level is given
-        to the unprivileged CouchDB user set on the server process.
-        This is achieved by creating a _security design document, see:
-        http://docs.couchdb.org/en/latest/api/database/security.html
-
-        :param database_security: security configuration parsed from conf file
-        :type cache: dict
-        """
-        security_config = security_config or {}
-        security = self._database.resource.get_json('_security')[2]
-        security['members'] = {'names': [], 'roles': []}
-        security['members']['names'] = security_config.get('members',
-                                                           ['soledad'])
-        security['members']['roles'] = security_config.get('members_roles', [])
-        security['admins'] = {'names': [], 'roles': []}
-        security['admins']['names'] = security_config.get('admins', [])
-        security['admins']['roles'] = security_config.get('admins_roles', [])
-        self._database.resource.put_json('_security', body=security)
-
-    def delete_database(self):
-        """
-        Delete a U1DB CouchDB database.
-        """
-        with couch_server(self._url) as server:
-            del(server[self._dbname])
-
-    def set_replica_uid(self, replica_uid):
-        """
-        Force the replica uid to be set.
-
-        :param replica_uid: The new replica uid.
-        :type replica_uid: str
-        """
-        try:
-            # set on existent config document
-            doc = self._database['u1db_config']
-            doc['replica_uid'] = replica_uid
-        except ResourceNotFound:
-            # or create the config document
-            doc = {
-                '_id': 'u1db_config',
-                'replica_uid': replica_uid,
-            }
-        self._database.save(doc)
-
-    def get_replica_uid(self):
-        """
-        Get the replica uid.
-
-        :return: The replica uid.
-        :rtype: str
-        """
-        try:
-            # grab replica_uid from server
-            doc = self._database['u1db_config']
-            replica_uid = doc['replica_uid']
-            return replica_uid
-        except ResourceNotFound:
-            # create a unique replica_uid
-            replica_uid = uuid.uuid4().hex
-            self.set_replica_uid(replica_uid)
-            return replica_uid
-
-    def close(self):
-        self._database = None
-
-    def get_all_docs(self, include_deleted=False):
-        """
-        Get the JSON content for all documents in the database.
-
-        :param include_deleted: If set to True, deleted documents will be
-                                returned with empty content. Otherwise deleted
-                                documents will not be included in the results.
-        :type include_deleted: bool
-
-        :return: (generation, [CouchDocument])
-            The current generation of the database, followed by a list of all
-            the documents in the database.
-        :rtype: (int, [CouchDocument])
-        """
-
-        generation, _ = self._get_generation_info()
-        results = list(self.get_docs(self._database,
-                                     include_deleted=include_deleted))
-        return (generation, results)
-
-    def get_docs(self, doc_ids, check_for_conflicts=True,
-                 include_deleted=False):
-        """
-        Get the JSON content for many documents.
-
-        :param doc_ids: A list of document identifiers or None for all.
-        :type doc_ids: list
-        :param check_for_conflicts: If set to False, then the conflict check
-                                    will be skipped, and 'None' will be
-                                    returned instead of True/False.
-        :type check_for_conflicts: bool
-        :param include_deleted: If set to True, deleted documents will be
-                                returned with empty content. Otherwise deleted
-                                documents will not be included in the results.
-        :return: iterable giving the Document object for each document id
-                 in matching doc_ids order.
-        :rtype: iterable
-        """
-        # Workaround for:
-        #
-        #   http://bugs.python.org/issue7980
-        #   https://leap.se/code/issues/5449
-        #
-        # python-couchdb uses time.strptime, which is not thread safe. In
-        # order to avoid the problem described on the issues above, we preload
-        # strptime here by evaluating the conversion of an arbitrary date.
-        # This will not be needed when/if we switch from python-couchdb to
-        # paisley.
-        time.strptime('Mar 8 1917', '%b %d %Y')
-        get_one = lambda doc_id: self._get_doc(doc_id, check_for_conflicts)
-        docs = [THREAD_POOL.apply_async(get_one, [doc_id])
-                for doc_id in doc_ids]
-        for doc in docs:
-            doc = doc.get()
-            if not doc or not include_deleted and doc.is_tombstone():
-                continue
-            yield doc
-
-    def _get_doc(self, doc_id, check_for_conflicts=False):
-        """
-        Extract the document from storage.
-
-        This can return None if the document doesn't exist.
-
-        :param doc_id: The unique document identifier
-        :type doc_id: str
-        :param check_for_conflicts: If set to False, then the conflict check
-                                    will be skipped.
-        :type check_for_conflicts: bool
-
-        :return: The document.
-        :rtype: CouchDocument
-        """
-        # get document with all attachments (u1db content and eventual
-        # conflicts)
-        try:
-            result = \
-                self._database.resource(doc_id).get_json(
-                    attachments=True)[2]
-        except ResourceNotFound:
-            return None
-        return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts)
-
-    def __parse_doc_from_couch(self, result, doc_id,
-                               check_for_conflicts=False):
-        # restrict to u1db documents
-        if 'u1db_rev' not in result:
-            return None
-        doc = CouchDocument(doc_id, result['u1db_rev'])
-        # set contents or make tombstone
-        if '_attachments' not in result \
-                or 'u1db_content' not in result['_attachments']:
-            doc.make_tombstone()
-        else:
-            doc.content = json.loads(
-                binascii.a2b_base64(
-                    result['_attachments']['u1db_content']['data']))
-        # determine if there are conflicts
-        if check_for_conflicts \
-                and '_attachments' in result \
-                and 'u1db_conflicts' in result['_attachments']:
-            doc.set_conflicts(
-                self._build_conflicts(
-                    doc.doc_id,
-                    json.loads(binascii.a2b_base64(
-                        result['_attachments']['u1db_conflicts']['data']))))
-        # store couch revision
-        doc.couch_rev = result['_rev']
-        # store transactions
-        doc.transactions = result['u1db_transactions']
-        return doc
-
-    def _build_conflicts(self, doc_id, attached_conflicts):
-        """
-        Build the conflicted documents list from the conflicts attachment
-        fetched from a couch document.
-
-        :param attached_conflicts: The document's conflicts as fetched from a
-                                   couch document attachment.
-        :type attached_conflicts: dict
-        """
-        conflicts = []
-        for doc_rev, content in attached_conflicts:
-            doc = CouchDocument(doc_id, doc_rev)
-            if content is None:
-                doc.make_tombstone()
-            else:
-                doc.content = content
-            conflicts.append(doc)
-        return conflicts
-
-    def _get_trans_id_for_gen(self, generation):
-        """
-        Get the transaction id corresponding to a particular generation.
-
-        :param generation: The generation for which to get the transaction id.
-        :type generation: int
-
-        :return: The transaction id for C{generation}.
-        :rtype: str
-
-        :raise InvalidGeneration: Raised when the generation does not exist.
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
-        """
-        if generation == 0:
-            return ''
-        # query a couch list function
-        ddoc_path = [
-            '_design', 'transactions', '_list', 'trans_id_for_gen', 'log'
-        ]
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json(gen=generation)
-            if response[2] == {}:
-                raise InvalidGeneration
-            return response[2]['transaction_id']
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-        except ServerError as e:
-            raise_server_error(e, ddoc_path)
-
-    def _get_replica_gen_and_trans_id(self, other_replica_uid):
-        """
-        Return the last known generation and transaction id for the other db
-        replica.
-
-        When you do a synchronization with another replica, the Database keeps
-        track of what generation the other database replica was at, and what
-        the associated transaction id was.  This is used to determine what data
-        needs to be sent, and if two databases are claiming to be the same
-        replica.
-
-        :param other_replica_uid: The identifier for the other replica.
-        :type other_replica_uid: str
-
-        :return: A tuple containing the generation and transaction id we
-                 encountered during synchronization. If we've never
-                 synchronized with the replica, this is (0, '').
-        :rtype: (int, str)
-        """
-        doc_id = 'u1db_sync_%s' % other_replica_uid
-        try:
-            doc = self._database[doc_id]
-        except ResourceNotFound:
-            doc = {
-                '_id': doc_id,
-                'generation': 0,
-                'transaction_id': '',
-            }
-            self._database.save(doc)
-        result = doc['generation'], doc['transaction_id']
-        return result
-
-    def get_doc_conflicts(self, doc_id, couch_rev=None):
-        """
-        Get the conflicted versions of a document.
-
-        If the C{couch_rev} parameter is not None, conflicts for a specific
-        document's couch revision are returned.
-
-        :param couch_rev: The couch document revision.
-        :type couch_rev: str
-
-        :return: A list of conflicted versions of the document.
-        :rtype: list
-        """
-        # request conflicts attachment from server
-        params = {}
-        conflicts = []
-        if couch_rev is not None:
-            params['rev'] = couch_rev  # restric document's couch revision
-        else:
-            # TODO: move into resource logic!
-            first_entry = self._get_doc(doc_id, check_for_conflicts=True)
-            conflicts.append(first_entry)
-        resource = self._database.resource(doc_id, 'u1db_conflicts')
-        try:
-            response = resource.get_json(**params)
-            return conflicts + self._build_conflicts(
-                doc_id, json.loads(response[2].read()))
-        except ResourceNotFound:
-            return []
-
-    def _do_set_replica_gen_and_trans_id(
-            self, other_replica_uid, other_generation, other_transaction_id,
-            number_of_docs=None, doc_idx=None, sync_id=None):
-        """
-        Set the last-known generation and transaction id for the other
-        database replica.
-
-        We have just performed some synchronization, and we want to track what
-        generation the other replica was at. See also
-        _get_replica_gen_and_trans_id.
-
-        :param other_replica_uid: The U1DB identifier for the other replica.
-        :type other_replica_uid: str
-        :param other_generation: The generation number for the other replica.
-        :type other_generation: int
-        :param other_transaction_id: The transaction id associated with the
-                                     generation.
-        :type other_transaction_id: str
-        :param number_of_docs: The total amount of documents sent on this sync
-                               session.
-        :type number_of_docs: int
-        :param doc_idx: The index of the current document being sent.
-        :type doc_idx: int
-        :param sync_id: The id of the current sync session.
-        :type sync_id: str
-        """
-        doc_id = 'u1db_sync_%s' % other_replica_uid
-        try:
-            doc = self._database[doc_id]
-        except ResourceNotFound:
-            doc = {'_id': doc_id}
-        doc['generation'] = other_generation
-        doc['transaction_id'] = other_transaction_id
-        self._database.save(doc)
-
-    def _get_transaction_log(self):
-        """
-        This is only for the test suite, it is not part of the api.
-
-        :return: The complete transaction log.
-        :rtype: [(str, str)]
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
-        """
-        # query a couch view
-        ddoc_path = ['_design', 'transactions', '_view', 'log']
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json()
-            return map(
-                lambda row: (row['id'], row['value']),
-                response[2]['rows'])
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-
-    def whats_changed(self, old_generation=0):
-        """
-        Return a list of documents that have changed since old_generation.
-
-        :param old_generation: The generation of the database in the old
-                               state.
-        :type old_generation: int
-
-        :return: (generation, trans_id, [(doc_id, generation, trans_id),...])
-                 The current generation of the database, its associated
-                 transaction id, and a list of of changed documents since
-                 old_generation, represented by tuples with for each document
-                 its doc_id and the generation and transaction id corresponding
-                 to the last intervening change and sorted by generation (old
-                 changes first)
-        :rtype: (int, str, [(str, int, str)])
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
-        """
-        # query a couch list function
-        ddoc_path = [
-            '_design', 'transactions', '_list', 'whats_changed', 'log'
-        ]
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json(old_gen=old_generation)
-            results = map(
-                lambda row:
-                    (row['generation'], row['doc_id'], row['transaction_id']),
-                response[2]['transactions'])
-            results.reverse()
-            cur_gen = old_generation
-            seen = set()
-            changes = []
-            newest_trans_id = ''
-            for generation, doc_id, trans_id in results:
-                if doc_id not in seen:
-                    changes.append((doc_id, generation, trans_id))
-                    seen.add(doc_id)
-            if changes:
-                cur_gen = changes[0][1]  # max generation
-                newest_trans_id = changes[0][2]
-                changes.reverse()
-            else:
-                cur_gen, newest_trans_id = self._get_generation_info()
-
-            return cur_gen, newest_trans_id, changes
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-        except ServerError as e:
-            raise_server_error(e, ddoc_path)
-
-    def _get_generation_info(self):
-        """
-        Return the current generation.
-
-        :return: A tuple containing the current generation and transaction id.
-        :rtype: (int, str)
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
-        """
-        # query a couch list function
-        ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json()
-            return (response[2]['generation'], response[2]['transaction_id'])
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-        except ServerError as e:
-            raise_server_error(e, ddoc_path)
-
-    def save_document(self, old_doc, doc, transaction_id):
-        """
-        Put the document in the Couch backend database.
-
-        Note that C{old_doc} must have been fetched with the parameter
-        C{check_for_conflicts} equal to True, so we can properly update the
-        new document using the conflict information from the old one.
-
-        :param old_doc: The old document version.
-        :type old_doc: CouchDocument
-        :param doc: The document to be put.
-        :type doc: CouchDocument
-
-        :raise RevisionConflict: Raised when trying to update a document but
-                                 couch revisions mismatch.
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
-        """
-        attachments = {}  # we save content and conflicts as attachments
-        parts = []  # and we put it using couch's multipart PUT
-        # save content as attachment
-        if doc.is_tombstone() is False:
-            content = doc.get_json()
-            attachments['u1db_content'] = {
-                'follows': True,
-                'content_type': 'application/octet-stream',
-                'length': len(content),
-            }
-            parts.append(content)
-        # save conflicts as attachment
-        if doc.has_conflicts is True:
-            conflicts = json.dumps(
-                map(lambda cdoc: (cdoc.rev, cdoc.content),
-                    doc.get_conflicts()))
-            attachments['u1db_conflicts'] = {
-                'follows': True,
-                'content_type': 'application/octet-stream',
-                'length': len(conflicts),
-            }
-            parts.append(conflicts)
-        # store old transactions, if any
-        transactions = old_doc.transactions[:] if old_doc is not None else []
-        # create a new transaction id and timestamp it so the transaction log
-        # is consistent when querying the database.
-        transactions.append(
-            # here we store milliseconds to keep consistent with javascript
-            # Date.prototype.getTime() which was used before inside a couchdb
-            # update handler.
-            (int(time.time() * 1000),
-             transaction_id))
-        # build the couch document
-        couch_doc = {
-            '_id': doc.doc_id,
-            'u1db_rev': doc.rev,
-            'u1db_transactions': transactions,
-            '_attachments': attachments,
-        }
-        # if we are updating a doc we have to add the couch doc revision
-        if old_doc is not None:
-            couch_doc['_rev'] = old_doc.couch_rev
-        # prepare the multipart PUT
-        buf = StringIO()
-        headers = {}
-        envelope = MultipartWriter(buf, headers=headers, subtype='related')
-        envelope.add('application/json', json.dumps(couch_doc))
-        for part in parts:
-            envelope.add('application/octet-stream', part)
-        envelope.close()
-        # try to save and fail if there's a revision conflict
-        try:
-            resource = self._new_resource()
-            resource.put_json(
-                doc.doc_id, body=str(buf.getvalue()), headers=headers)
-        except ResourceConflict:
-            raise RevisionConflict()
-        return transactions[-1][1]
-
-    def _new_resource(self, *path):
-        """
-        Return a new resource for accessing a couch database.
-
-        :return: A resource for accessing a couch database.
-        :rtype: couchdb.http.Resource
-        """
-        # Workaround for: https://leap.se/code/issues/5448
-        url = couch_urljoin(self._database.resource.url, *path)
-        resource = Resource(url, Session(timeout=COUCH_TIMEOUT))
-        resource.credentials = self._database.resource.credentials
-        resource.headers = self._database.resource.headers.copy()
-        return resource
-
-
-class CouchSyncTarget(CommonSyncTarget):
-
-    """
-    Functionality for using a SoledadBackend as a synchronization target.
-    """
-
-    def get_sync_info(self, source_replica_uid):
-        source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id(
-            source_replica_uid)
-        my_gen, my_trans_id = self._db._get_generation_info()
-        return (
-            self._db._replica_uid, my_gen, my_trans_id, source_gen,
-            source_trans_id)
-
-    def record_sync_info(self, source_replica_uid, source_replica_generation,
-                         source_replica_transaction_id):
-        if self._trace_hook:
-            self._trace_hook('record_sync_info')
-        self._db._set_replica_gen_and_trans_id(
-            source_replica_uid, source_replica_generation,
-            source_replica_transaction_id)
-
-
-def is_db_name_valid(name):
-    """
-    Validate a user database using a regular expression.
-
-    :param name: database name.
-    :type name: str
-
-    :return: boolean for name vailidity
-    :rtype: bool
-    """
-    db_name_regex = "^user-[a-f0-9]+$"
-    return re.match(db_name_regex, name) is not None
-
-
-class CouchServerState(ServerState):
-
-    """
-    Inteface of the WSGI server with the CouchDB backend.
-    """
-
-    def __init__(self, couch_url, create_cmd=None):
-        """
-        Initialize the couch server state.
-
-        :param couch_url: The URL for the couch database.
-        :type couch_url: str
-        """
-        self.couch_url = couch_url
-        self.create_cmd = create_cmd
-
-    def open_database(self, dbname):
-        """
-        Open a couch database.
-
-        :param dbname: The name of the database to open.
-        :type dbname: str
-
-        :return: The SoledadBackend object.
-        :rtype: SoledadBackend
-        """
-        url = urljoin(self.couch_url, dbname)
-        db = SoledadBackend.open_database(url, create=False,
-                                          ensure_ddocs=False)
-        return db
-
-    def ensure_database(self, dbname):
-        """
-        Ensure couch database exists.
-
-        :param dbname: The name of the database to ensure.
-        :type dbname: str
-
-        :raise Unauthorized: If disabled or other error was raised.
-
-        :return: The SoledadBackend object and its replica_uid.
-        :rtype: (SoledadBackend, str)
-        """
-        if not self.create_cmd:
-            raise Unauthorized()
-        else:
-            code, out = exec_validated_cmd(self.create_cmd, dbname,
-                                           validator=is_db_name_valid)
-            if code is not 0:
-                logger.error("""
-                    Error while creating database (%s) with (%s) command.
-                    Output: %s
-                    Exit code: %d
-                    """ % (dbname, self.create_cmd, out, code))
-                raise Unauthorized()
-        db = self.open_database(dbname)
-        return db, db.replica_uid
-
-    def delete_database(self, dbname):
-        """
-        Delete couch database.
-
-        :param dbname: The name of the database to delete.
-        :type dbname: str
-
-        :raise Unauthorized: Always, because Soledad server is not allowed to
-                             delete databases.
-        """
-        raise Unauthorized()
diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
new file mode 100644
index 00000000..727f033f
--- /dev/null
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -0,0 +1,746 @@
+# -*- coding: utf-8 -*-
+# couch.py
+# Copyright (C) 2013 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""A U1DB backend that uses CouchDB as its persistence layer."""
+
+
+import json
+import re
+import uuid
+import binascii
+import time
+
+
+from StringIO import StringIO
+from urlparse import urljoin
+from contextlib import contextmanager
+from multiprocessing.pool import ThreadPool
+
+
+from couchdb.client import Server, Database
+from couchdb.multipart import MultipartWriter
+from couchdb.http import (
+    ResourceConflict,
+    ResourceNotFound,
+    ServerError,
+    Session,
+    urljoin as couch_urljoin,
+    Resource,
+)
+from u1db.errors import (
+    DatabaseDoesNotExist,
+    InvalidGeneration,
+    RevisionConflict,
+)
+from u1db.remote import http_app
+
+
+from leap.soledad.common import ddocs
+from leap.soledad.common.errors import raise_server_error
+from leap.soledad.common.errors import raise_missing_design_doc_error
+from leap.soledad.common.errors import InvalidURLError
+from leap.soledad.common.document import CouchDocument
+from leap.soledad.common.backend import SoledadBackend
+
+
+COUCH_TIMEOUT = 120  # timeout for transfers between Soledad server and Couch
+
+
+def list_users_dbs(couch_url):
+    """
+    Retrieves a list with all databases that starts with 'user-' on CouchDB.
+    Those databases belongs to users. So, the list will contain all the
+    database names in the form of 'user-{uuid4}'.
+
+    :param couch_url: The couch url with needed credentials
+    :type couch_url: str
+
+    :return: The list of all database names from users.
+    :rtype: [str]
+    """
+    with couch_server(couch_url) as server:
+        users = [dbname for dbname in server if dbname.startswith('user-')]
+    return users
+
+
+# monkey-patch the u1db http app to use CouchDocument
+http_app.Document = CouchDocument
+
+
+@contextmanager
+def couch_server(url):
+    """
+    Provide a connection to a couch server and cleanup after use.
+
+    For database creation and deletion we use an ephemeral connection to the
+    couch server. That connection has to be properly closed, so we provide it
+    as a context manager.
+
+    :param url: The URL of the Couch server.
+    :type url: str
+    """
+    session = Session(timeout=COUCH_TIMEOUT)
+    server = Server(url=url, full_commit=False, session=session)
+    yield server
+
+
+THREAD_POOL = ThreadPool(20)
+
+
+class CouchDatabase(object):
+    """
+    Holds CouchDB related code.
+    This class gives methods to encapsulate database operations and hide
+    CouchDB details from backend code.
+    """
+
+    @classmethod
+    def open_database(cls, url, create, ensure_ddocs=False, replica_uid=None,
+                      database_security=None):
+        """
+        Open a U1DB database using CouchDB as backend.
+
+        :param url: the url of the database replica
+        :type url: str
+        :param create: should the replica be created if it does not exist?
+        :type create: bool
+        :param replica_uid: an optional unique replica identifier
+        :type replica_uid: str
+        :param ensure_ddocs: Ensure that the design docs exist on server.
+        :type ensure_ddocs: bool
+
+        :return: the database instance
+        :rtype: SoledadBackend
+        """
+        # get database from url
+        m = re.match('(^https?://[^/]+)/(.+)$', url)
+        if not m:
+            raise InvalidURLError
+        url = m.group(1)
+        dbname = m.group(2)
+        with couch_server(url) as server:
+            try:
+                server[dbname]
+            except ResourceNotFound:
+                if not create:
+                    raise DatabaseDoesNotExist()
+                server.create(dbname)
+        db = cls(url,
+                 dbname, ensure_ddocs=ensure_ddocs,
+                 database_security=database_security)
+        return SoledadBackend(
+            db, replica_uid=replica_uid)
+
+    def __init__(self, url, dbname, ensure_ddocs=True,
+                 database_security=None):
+        self._session = Session(timeout=COUCH_TIMEOUT)
+        self._url = url
+        self._dbname = dbname
+        self._database = Database(
+            urljoin(url, dbname),
+            self._session)
+        self._database.info()
+        if ensure_ddocs:
+            self.ensure_ddocs_on_db()
+            self.ensure_security_ddoc(database_security)
+
+    def ensure_ddocs_on_db(self):
+        """
+        Ensure that the design documents used by the backend exist on the
+        couch database.
+        """
+        for ddoc_name in ['docs', 'syncs', 'transactions']:
+            try:
+                self._database.resource('_design',
+                                        ddoc_name, '_info').get_json()
+            except ResourceNotFound:
+                ddoc = json.loads(
+                    binascii.a2b_base64(
+                        getattr(ddocs, ddoc_name)))
+                self._database.save(ddoc)
+
+    def ensure_security_ddoc(self, security_config=None):
+        """
+        Make sure that only soledad user is able to access this database as
+        an unprivileged member, meaning that administration access will
+        be forbidden even inside an user database.
+        The goal is to make sure that only the lowest access level is given
+        to the unprivileged CouchDB user set on the server process.
+        This is achieved by creating a _security design document, see:
+        http://docs.couchdb.org/en/latest/api/database/security.html
+
+        :param database_security: security configuration parsed from conf file
+        :type cache: dict
+        """
+        security_config = security_config or {}
+        security = self._database.resource.get_json('_security')[2]
+        security['members'] = {'names': [], 'roles': []}
+        security['members']['names'] = security_config.get('members',
+                                                           ['soledad'])
+        security['members']['roles'] = security_config.get('members_roles', [])
+        security['admins'] = {'names': [], 'roles': []}
+        security['admins']['names'] = security_config.get('admins', [])
+        security['admins']['roles'] = security_config.get('admins_roles', [])
+        self._database.resource.put_json('_security', body=security)
+
+    def delete_database(self):
+        """
+        Delete a U1DB CouchDB database.
+        """
+        with couch_server(self._url) as server:
+            del(server[self._dbname])
+
+    def set_replica_uid(self, replica_uid):
+        """
+        Force the replica uid to be set.
+
+        :param replica_uid: The new replica uid.
+        :type replica_uid: str
+        """
+        try:
+            # set on existent config document
+            doc = self._database['u1db_config']
+            doc['replica_uid'] = replica_uid
+        except ResourceNotFound:
+            # or create the config document
+            doc = {
+                '_id': 'u1db_config',
+                'replica_uid': replica_uid,
+            }
+        self._database.save(doc)
+
+    def get_replica_uid(self):
+        """
+        Get the replica uid.
+
+        :return: The replica uid.
+        :rtype: str
+        """
+        try:
+            # grab replica_uid from server
+            doc = self._database['u1db_config']
+            replica_uid = doc['replica_uid']
+            return replica_uid
+        except ResourceNotFound:
+            # create a unique replica_uid
+            replica_uid = uuid.uuid4().hex
+            self.set_replica_uid(replica_uid)
+            return replica_uid
+
+    def close(self):
+        self._database = None
+
+    def get_all_docs(self, include_deleted=False):
+        """
+        Get the JSON content for all documents in the database.
+
+        :param include_deleted: If set to True, deleted documents will be
+                                returned with empty content. Otherwise deleted
+                                documents will not be included in the results.
+        :type include_deleted: bool
+
+        :return: (generation, [CouchDocument])
+            The current generation of the database, followed by a list of all
+            the documents in the database.
+        :rtype: (int, [CouchDocument])
+        """
+
+        generation, _ = self._get_generation_info()
+        results = list(self.get_docs(self._database,
+                                     include_deleted=include_deleted))
+        return (generation, results)
+
+    def get_docs(self, doc_ids, check_for_conflicts=True,
+                 include_deleted=False):
+        """
+        Get the JSON content for many documents.
+
+        :param doc_ids: A list of document identifiers or None for all.
+        :type doc_ids: list
+        :param check_for_conflicts: If set to False, then the conflict check
+                                    will be skipped, and 'None' will be
+                                    returned instead of True/False.
+        :type check_for_conflicts: bool
+        :param include_deleted: If set to True, deleted documents will be
+                                returned with empty content. Otherwise deleted
+                                documents will not be included in the results.
+        :return: iterable giving the Document object for each document id
+                 in matching doc_ids order.
+        :rtype: iterable
+        """
+        # Workaround for:
+        #
+        #   http://bugs.python.org/issue7980
+        #   https://leap.se/code/issues/5449
+        #
+        # python-couchdb uses time.strptime, which is not thread safe. In
+        # order to avoid the problem described on the issues above, we preload
+        # strptime here by evaluating the conversion of an arbitrary date.
+        # This will not be needed when/if we switch from python-couchdb to
+        # paisley.
+        time.strptime('Mar 8 1917', '%b %d %Y')
+        get_one = lambda doc_id: self._get_doc(doc_id, check_for_conflicts)
+        docs = [THREAD_POOL.apply_async(get_one, [doc_id])
+                for doc_id in doc_ids]
+        for doc in docs:
+            doc = doc.get()
+            if not doc or not include_deleted and doc.is_tombstone():
+                continue
+            yield doc
+
+    def _get_doc(self, doc_id, check_for_conflicts=False):
+        """
+        Extract the document from storage.
+
+        This can return None if the document doesn't exist.
+
+        :param doc_id: The unique document identifier
+        :type doc_id: str
+        :param check_for_conflicts: If set to False, then the conflict check
+                                    will be skipped.
+        :type check_for_conflicts: bool
+
+        :return: The document.
+        :rtype: CouchDocument
+        """
+        # get document with all attachments (u1db content and eventual
+        # conflicts)
+        try:
+            result = \
+                self._database.resource(doc_id).get_json(
+                    attachments=True)[2]
+        except ResourceNotFound:
+            return None
+        return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts)
+
+    def __parse_doc_from_couch(self, result, doc_id,
+                               check_for_conflicts=False):
+        # restrict to u1db documents
+        if 'u1db_rev' not in result:
+            return None
+        doc = CouchDocument(doc_id, result['u1db_rev'])
+        # set contents or make tombstone
+        if '_attachments' not in result \
+                or 'u1db_content' not in result['_attachments']:
+            doc.make_tombstone()
+        else:
+            doc.content = json.loads(
+                binascii.a2b_base64(
+                    result['_attachments']['u1db_content']['data']))
+        # determine if there are conflicts
+        if check_for_conflicts \
+                and '_attachments' in result \
+                and 'u1db_conflicts' in result['_attachments']:
+            doc.set_conflicts(
+                self._build_conflicts(
+                    doc.doc_id,
+                    json.loads(binascii.a2b_base64(
+                        result['_attachments']['u1db_conflicts']['data']))))
+        # store couch revision
+        doc.couch_rev = result['_rev']
+        # store transactions
+        doc.transactions = result['u1db_transactions']
+        return doc
+
+    def _build_conflicts(self, doc_id, attached_conflicts):
+        """
+        Build the conflicted documents list from the conflicts attachment
+        fetched from a couch document.
+
+        :param attached_conflicts: The document's conflicts as fetched from a
+                                   couch document attachment.
+        :type attached_conflicts: dict
+        """
+        conflicts = []
+        for doc_rev, content in attached_conflicts:
+            doc = CouchDocument(doc_id, doc_rev)
+            if content is None:
+                doc.make_tombstone()
+            else:
+                doc.content = content
+            conflicts.append(doc)
+        return conflicts
+
+    def _get_trans_id_for_gen(self, generation):
+        """
+        Get the transaction id corresponding to a particular generation.
+
+        :param generation: The generation for which to get the transaction id.
+        :type generation: int
+
+        :return: The transaction id for C{generation}.
+        :rtype: str
+
+        :raise InvalidGeneration: Raised when the generation does not exist.
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        if generation == 0:
+            return ''
+        # query a couch list function
+        ddoc_path = [
+            '_design', 'transactions', '_list', 'trans_id_for_gen', 'log'
+        ]
+        res = self._database.resource(*ddoc_path)
+        try:
+            response = res.get_json(gen=generation)
+            if response[2] == {}:
+                raise InvalidGeneration
+            return response[2]['transaction_id']
+        except ResourceNotFound as e:
+            raise_missing_design_doc_error(e, ddoc_path)
+        except ServerError as e:
+            raise_server_error(e, ddoc_path)
+
+    def _get_replica_gen_and_trans_id(self, other_replica_uid):
+        """
+        Return the last known generation and transaction id for the other db
+        replica.
+
+        When you do a synchronization with another replica, the Database keeps
+        track of what generation the other database replica was at, and what
+        the associated transaction id was.  This is used to determine what data
+        needs to be sent, and if two databases are claiming to be the same
+        replica.
+
+        :param other_replica_uid: The identifier for the other replica.
+        :type other_replica_uid: str
+
+        :return: A tuple containing the generation and transaction id we
+                 encountered during synchronization. If we've never
+                 synchronized with the replica, this is (0, '').
+        :rtype: (int, str)
+        """
+        doc_id = 'u1db_sync_%s' % other_replica_uid
+        try:
+            doc = self._database[doc_id]
+        except ResourceNotFound:
+            doc = {
+                '_id': doc_id,
+                'generation': 0,
+                'transaction_id': '',
+            }
+            self._database.save(doc)
+        result = doc['generation'], doc['transaction_id']
+        return result
+
+    def get_doc_conflicts(self, doc_id, couch_rev=None):
+        """
+        Get the conflicted versions of a document.
+
+        If the C{couch_rev} parameter is not None, conflicts for a specific
+        document's couch revision are returned.
+
+        :param couch_rev: The couch document revision.
+        :type couch_rev: str
+
+        :return: A list of conflicted versions of the document.
+        :rtype: list
+        """
+        # request conflicts attachment from server
+        params = {}
+        conflicts = []
+        if couch_rev is not None:
+            params['rev'] = couch_rev  # restric document's couch revision
+        else:
+            # TODO: move into resource logic!
+            first_entry = self._get_doc(doc_id, check_for_conflicts=True)
+            conflicts.append(first_entry)
+        resource = self._database.resource(doc_id, 'u1db_conflicts')
+        try:
+            response = resource.get_json(**params)
+            return conflicts + self._build_conflicts(
+                doc_id, json.loads(response[2].read()))
+        except ResourceNotFound:
+            return []
+
+    def _do_set_replica_gen_and_trans_id(
+            self, other_replica_uid, other_generation, other_transaction_id,
+            number_of_docs=None, doc_idx=None, sync_id=None):
+        """
+        Set the last-known generation and transaction id for the other
+        database replica.
+
+        We have just performed some synchronization, and we want to track what
+        generation the other replica was at. See also
+        _get_replica_gen_and_trans_id.
+
+        :param other_replica_uid: The U1DB identifier for the other replica.
+        :type other_replica_uid: str
+        :param other_generation: The generation number for the other replica.
+        :type other_generation: int
+        :param other_transaction_id: The transaction id associated with the
+                                     generation.
+        :type other_transaction_id: str
+        :param number_of_docs: The total amount of documents sent on this sync
+                               session.
+        :type number_of_docs: int
+        :param doc_idx: The index of the current document being sent.
+        :type doc_idx: int
+        :param sync_id: The id of the current sync session.
+        :type sync_id: str
+        """
+        doc_id = 'u1db_sync_%s' % other_replica_uid
+        try:
+            doc = self._database[doc_id]
+        except ResourceNotFound:
+            doc = {'_id': doc_id}
+        doc['generation'] = other_generation
+        doc['transaction_id'] = other_transaction_id
+        self._database.save(doc)
+
+    def _get_transaction_log(self):
+        """
+        This is only for the test suite, it is not part of the api.
+
+        :return: The complete transaction log.
+        :rtype: [(str, str)]
+
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        # query a couch view
+        ddoc_path = ['_design', 'transactions', '_view', 'log']
+        res = self._database.resource(*ddoc_path)
+        try:
+            response = res.get_json()
+            return map(
+                lambda row: (row['id'], row['value']),
+                response[2]['rows'])
+        except ResourceNotFound as e:
+            raise_missing_design_doc_error(e, ddoc_path)
+
+    def whats_changed(self, old_generation=0):
+        """
+        Return a list of documents that have changed since old_generation.
+
+        :param old_generation: The generation of the database in the old
+                               state.
+        :type old_generation: int
+
+        :return: (generation, trans_id, [(doc_id, generation, trans_id),...])
+                 The current generation of the database, its associated
+                 transaction id, and a list of of changed documents since
+                 old_generation, represented by tuples with for each document
+                 its doc_id and the generation and transaction id corresponding
+                 to the last intervening change and sorted by generation (old
+                 changes first)
+        :rtype: (int, str, [(str, int, str)])
+
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        # query a couch list function
+        ddoc_path = [
+            '_design', 'transactions', '_list', 'whats_changed', 'log'
+        ]
+        res = self._database.resource(*ddoc_path)
+        try:
+            response = res.get_json(old_gen=old_generation)
+            results = map(
+                lambda row:
+                    (row['generation'], row['doc_id'], row['transaction_id']),
+                response[2]['transactions'])
+            results.reverse()
+            cur_gen = old_generation
+            seen = set()
+            changes = []
+            newest_trans_id = ''
+            for generation, doc_id, trans_id in results:
+                if doc_id not in seen:
+                    changes.append((doc_id, generation, trans_id))
+                    seen.add(doc_id)
+            if changes:
+                cur_gen = changes[0][1]  # max generation
+                newest_trans_id = changes[0][2]
+                changes.reverse()
+            else:
+                cur_gen, newest_trans_id = self._get_generation_info()
+
+            return cur_gen, newest_trans_id, changes
+        except ResourceNotFound as e:
+            raise_missing_design_doc_error(e, ddoc_path)
+        except ServerError as e:
+            raise_server_error(e, ddoc_path)
+
+    def _get_generation_info(self):
+        """
+        Return the current generation.
+
+        :return: A tuple containing the current generation and transaction id.
+        :rtype: (int, str)
+
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        # query a couch list function
+        ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
+        res = self._database.resource(*ddoc_path)
+        try:
+            response = res.get_json()
+            return (response[2]['generation'], response[2]['transaction_id'])
+        except ResourceNotFound as e:
+            raise_missing_design_doc_error(e, ddoc_path)
+        except ServerError as e:
+            raise_server_error(e, ddoc_path)
+
+    def save_document(self, old_doc, doc, transaction_id):
+        """
+        Put the document in the Couch backend database.
+
+        Note that C{old_doc} must have been fetched with the parameter
+        C{check_for_conflicts} equal to True, so we can properly update the
+        new document using the conflict information from the old one.
+
+        :param old_doc: The old document version.
+        :type old_doc: CouchDocument
+        :param doc: The document to be put.
+        :type doc: CouchDocument
+
+        :raise RevisionConflict: Raised when trying to update a document but
+                                 couch revisions mismatch.
+        :raise MissingDesignDocError: Raised when tried to access a missing
+                                      design document.
+        :raise MissingDesignDocListFunctionError: Raised when trying to access
+                                                  a missing list function on a
+                                                  design document.
+        :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                               missing named view on a design
+                                               document.
+        :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                             deleted design document.
+        :raise MissingDesignDocUnknownError: Raised when failed to access a
+                                             design document for an yet
+                                             unknown reason.
+        """
+        attachments = {}  # we save content and conflicts as attachments
+        parts = []  # and we put it using couch's multipart PUT
+        # save content as attachment
+        if doc.is_tombstone() is False:
+            content = doc.get_json()
+            attachments['u1db_content'] = {
+                'follows': True,
+                'content_type': 'application/octet-stream',
+                'length': len(content),
+            }
+            parts.append(content)
+        # save conflicts as attachment
+        if doc.has_conflicts is True:
+            conflicts = json.dumps(
+                map(lambda cdoc: (cdoc.rev, cdoc.content),
+                    doc.get_conflicts()))
+            attachments['u1db_conflicts'] = {
+                'follows': True,
+                'content_type': 'application/octet-stream',
+                'length': len(conflicts),
+            }
+            parts.append(conflicts)
+        # store old transactions, if any
+        transactions = old_doc.transactions[:] if old_doc is not None else []
+        # create a new transaction id and timestamp it so the transaction log
+        # is consistent when querying the database.
+        transactions.append(
+            # here we store milliseconds to keep consistent with javascript
+            # Date.prototype.getTime() which was used before inside a couchdb
+            # update handler.
+            (int(time.time() * 1000),
+             transaction_id))
+        # build the couch document
+        couch_doc = {
+            '_id': doc.doc_id,
+            'u1db_rev': doc.rev,
+            'u1db_transactions': transactions,
+            '_attachments': attachments,
+        }
+        # if we are updating a doc we have to add the couch doc revision
+        if old_doc is not None:
+            couch_doc['_rev'] = old_doc.couch_rev
+        # prepare the multipart PUT
+        buf = StringIO()
+        headers = {}
+        envelope = MultipartWriter(buf, headers=headers, subtype='related')
+        envelope.add('application/json', json.dumps(couch_doc))
+        for part in parts:
+            envelope.add('application/octet-stream', part)
+        envelope.close()
+        # try to save and fail if there's a revision conflict
+        try:
+            resource = self._new_resource()
+            resource.put_json(
+                doc.doc_id, body=str(buf.getvalue()), headers=headers)
+        except ResourceConflict:
+            raise RevisionConflict()
+        return transactions[-1][1]
+
+    def _new_resource(self, *path):
+        """
+        Return a new resource for accessing a couch database.
+
+        :return: A resource for accessing a couch database.
+        :rtype: couchdb.http.Resource
+        """
+        # Workaround for: https://leap.se/code/issues/5448
+        url = couch_urljoin(self._database.resource.url, *path)
+        resource = Resource(url, Session(timeout=COUCH_TIMEOUT))
+        resource.credentials = self._database.resource.credentials
+        resource.headers = self._database.resource.headers.copy()
+        return resource
diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py
new file mode 100644
index 00000000..39d49fa0
--- /dev/null
+++ b/common/src/leap/soledad/common/couch/state.py
@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+# state.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Server state using CouchDatabase as backend.
+"""
+import re
+import logging
+from urlparse import urljoin
+
+from u1db.remote.server_state import ServerState
+from leap.soledad.common.command import exec_validated_cmd
+from leap.soledad.common.couch import CouchDatabase
+from u1db.errors import Unauthorized
+
+
+logger = logging.getLogger(__name__)
+
+
+def is_db_name_valid(name):
+    """
+    Validate a user database using a regular expression.
+
+    :param name: database name.
+    :type name: str
+
+    :return: boolean for name vailidity
+    :rtype: bool
+    """
+    db_name_regex = "^user-[a-f0-9]+$"
+    return re.match(db_name_regex, name) is not None
+
+
+class CouchServerState(ServerState):
+
+    """
+    Inteface of the WSGI server with the CouchDB backend.
+    """
+
+    def __init__(self, couch_url, create_cmd=None):
+        """
+        Initialize the couch server state.
+
+        :param couch_url: The URL for the couch database.
+        :type couch_url: str
+        """
+        self.couch_url = couch_url
+        self.create_cmd = create_cmd
+
+    def open_database(self, dbname):
+        """
+        Open a couch database.
+
+        :param dbname: The name of the database to open.
+        :type dbname: str
+
+        :return: The SoledadBackend object.
+        :rtype: SoledadBackend
+        """
+        url = urljoin(self.couch_url, dbname)
+        db = CouchDatabase.open_database(url, create=False, ensure_ddocs=False)
+        return db
+
+    def ensure_database(self, dbname):
+        """
+        Ensure couch database exists.
+
+        :param dbname: The name of the database to ensure.
+        :type dbname: str
+
+        :raise Unauthorized: If disabled or other error was raised.
+
+        :return: The SoledadBackend object and its replica_uid.
+        :rtype: (SoledadBackend, str)
+        """
+        if not self.create_cmd:
+            raise Unauthorized()
+        else:
+            code, out = exec_validated_cmd(self.create_cmd, dbname,
+                                           validator=is_db_name_valid)
+            if code is not 0:
+                logger.error("""
+                    Error while creating database (%s) with (%s) command.
+                    Output: %s
+                    Exit code: %d
+                    """ % (dbname, self.create_cmd, out, code))
+                raise Unauthorized()
+        db = self.open_database(dbname)
+        return db, db.replica_uid
+
+    def delete_database(self, dbname):
+        """
+        Delete couch database.
+
+        :param dbname: The name of the database to delete.
+        :type dbname: str
+
+        :raise Unauthorized: Always, because Soledad server is not allowed to
+                             delete databases.
+        """
+        raise Unauthorized()
diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py
index ea9d6ce4..84d8d813 100644
--- a/common/src/leap/soledad/common/errors.py
+++ b/common/src/leap/soledad/common/errors.py
@@ -266,4 +266,4 @@ def raise_server_error(exc, ddoc_path):
         if 'point is undefined' in exc.message[1][1]:
             raise MissingDesignDocListFunctionError
     # other errors are unknown for now
-    raise DesignDocUnknownError(path)
+    raise DesignDocUnknownError("%s: %s" % (path, str(exc.message)))
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index b3536a6a..1634d30c 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -36,6 +36,7 @@ from u1db import SyncTarget
 from u1db import vectorclock
 
 from leap.soledad.common import couch
+from leap.soledad.common import backend
 from leap.soledad.common import errors
 
 from leap.soledad.common.tests import u1db_tests as tests
@@ -56,7 +57,7 @@ from u1db.backends.inmemory import InMemoryIndex
 class TestCouchBackendImpl(CouchDBTestCase):
 
     def test__allocate_doc_id(self):
-        db = couch.SoledadBackend.open_database(
+        db = couch.CouchDatabase.open_database(
             urljoin(
                 'http://localhost:' + str(self.couch_port),
                 ('test-%s' % uuid4().hex)
@@ -78,7 +79,7 @@ class TestCouchBackendImpl(CouchDBTestCase):
 def make_couch_database_for_test(test, replica_uid):
     port = str(test.couch_port)
     dbname = ('test-%s' % uuid4().hex)
-    db = couch.SoledadBackend.open_database(
+    db = couch.CouchDatabase.open_database(
         urljoin('http://localhost:' + port, dbname),
         create=True,
         replica_uid=replica_uid or 'test',
@@ -91,7 +92,7 @@ def copy_couch_database_for_test(test, db):
     port = str(test.couch_port)
     couch_url = 'http://localhost:' + port
     new_dbname = db._dbname + '_copy'
-    new_db = couch.SoledadBackend.open_database(
+    new_db = couch.CouchDatabase.open_database(
         urljoin(couch_url, new_dbname),
         create=True,
         replica_uid=db._replica_uid or 'test')
@@ -529,89 +530,6 @@ class SoledadBackendSyncTargetTests(
         self.st.record_sync_info('replica', 0, 'T-sid')
         self.assertEqual(expected, called)
 
-
-# The following tests need that the database have an index, so we fake one.
-
-class IndexedSoledadBackend(couch.SoledadBackend):
-
-    def __init__(self, db, replica_uid=None):
-        old_class.__init__(self, db, replica_uid=replica_uid)
-        self._indexes = {}
-
-    def _put_doc(self, old_doc, doc):
-        for index in self._indexes.itervalues():
-            if old_doc is not None and not old_doc.is_tombstone():
-                index.remove_json(old_doc.doc_id, old_doc.get_json())
-            if not doc.is_tombstone():
-                index.add_json(doc.doc_id, doc.get_json())
-        old_class._put_doc(self, old_doc, doc)
-
-    def create_index(self, index_name, *index_expressions):
-        if index_name in self._indexes:
-            if self._indexes[index_name]._definition == list(
-                    index_expressions):
-                return
-            raise u1db_errors.IndexNameTakenError
-        index = InMemoryIndex(index_name, list(index_expressions))
-        _, all_docs = self.get_all_docs()
-        for doc in all_docs:
-            index.add_json(doc.doc_id, doc.get_json())
-        self._indexes[index_name] = index
-
-    def delete_index(self, index_name):
-        del self._indexes[index_name]
-
-    def list_indexes(self):
-        definitions = []
-        for idx in self._indexes.itervalues():
-            definitions.append((idx._name, idx._definition))
-        return definitions
-
-    def get_from_index(self, index_name, *key_values):
-        try:
-            index = self._indexes[index_name]
-        except KeyError:
-            raise u1db_errors.IndexDoesNotExist
-        doc_ids = index.lookup(key_values)
-        result = []
-        for doc_id in doc_ids:
-            result.append(self._get_doc(doc_id, check_for_conflicts=True))
-        return result
-
-    def get_range_from_index(self, index_name, start_value=None,
-                             end_value=None):
-        """Return all documents with key values in the specified range."""
-        try:
-            index = self._indexes[index_name]
-        except KeyError:
-            raise u1db_errors.IndexDoesNotExist
-        if isinstance(start_value, basestring):
-            start_value = (start_value,)
-        if isinstance(end_value, basestring):
-            end_value = (end_value,)
-        doc_ids = index.lookup_range(start_value, end_value)
-        result = []
-        for doc_id in doc_ids:
-            result.append(self._get_doc(doc_id, check_for_conflicts=True))
-        return result
-
-    def get_index_keys(self, index_name):
-        try:
-            index = self._indexes[index_name]
-        except KeyError:
-            raise u1db_errors.IndexDoesNotExist
-        keys = index.keys()
-        # XXX inefficiency warning
-        return list(set([tuple(key.split('\x01')) for key in keys]))
-
-
-# monkey patch SoledadBackend (once) to include virtual indexes
-if getattr(couch.SoledadBackend, '_old_class', None) is None:
-    old_class = couch.SoledadBackend
-    IndexedSoledadBackend._old_class = old_class
-    couch.SoledadBackend = IndexedSoledadBackend
-
-
 sync_scenarios = []
 for name, scenario in COUCH_SCENARIOS:
     scenario = dict(scenario)
@@ -921,7 +839,6 @@ class SoledadBackendSyncTests(
         self.db1 = self.create_database('test1', 'source')
         self.db2 = self.create_database('test2', 'target')
         doc = self.db2.create_doc_from_json(simple_doc)
-        self.db1.create_index('test-idx', 'key')
         self.assertEqual(0, self.sync(self.db1, self.db2))
         self.assertGetDoc(self.db1, doc.doc_id, doc.rev, simple_doc, False)
         self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0])
@@ -931,7 +848,7 @@ class SoledadBackendSyncTests(
             {'receive': {'docs': [], 'last_known_gen': 0},
              'return': {'docs': [(doc.doc_id, doc.rev)],
                         'last_gen': 1}})
-        self.assertEqual([doc], self.db1.get_from_index('test-idx', 'value'))
+        self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False)
 
     def test_sync_pulling_doesnt_update_other_if_changed(self):
         self.db1 = self.create_database('test1', 'source')
@@ -1020,7 +937,7 @@ class SoledadBackendSyncTests(
         doc1 = self.db1.create_doc_from_json(simple_doc)
         doc_id = doc1.doc_id
         doc1_rev = doc1.rev
-        self.db1.create_index('test-idx', 'key')
+        # self.db1.create_index('test-idx', 'key')
         new_doc = '{"key": "altval"}'
         doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id)
         doc2_rev = doc2.rev
@@ -1036,18 +953,18 @@ class SoledadBackendSyncTests(
         self.assertTransactionLog([doc_id, doc_id], self.db1)
         self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True)
         self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False)
-        from_idx = self.db1.get_from_index('test-idx', 'altval')[0]
-        self.assertEqual(doc2.doc_id, from_idx.doc_id)
-        self.assertEqual(doc2.rev, from_idx.rev)
-        self.assertTrue(from_idx.has_conflicts)
-        self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
+        # soledad doesnt support index due to encryption
+        # from_idx = self.db1.get_from_index('test-idx', 'altval')[0]
+        # self.assertEqual(doc2.doc_id, from_idx.doc_id)
+        # self.assertEqual(doc2.rev, from_idx.rev)
+        # self.assertTrue(from_idx.has_conflicts)
+        # self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
 
     def test_sync_sees_remote_delete_conflicted(self):
         self.db1 = self.create_database('test1', 'source')
         self.db2 = self.create_database('test2', 'target')
         doc1 = self.db1.create_doc_from_json(simple_doc)
         doc_id = doc1.doc_id
-        self.db1.create_index('test-idx', 'key')
         self.sync(self.db1, self.db2)
         doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json())
         new_doc = '{"key": "altval"}'
@@ -1067,7 +984,6 @@ class SoledadBackendSyncTests(
         self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True)
         self.assertGetDocIncludeDeleted(
             self.db2, doc_id, doc2.rev, None, False)
-        self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
 
     def test_sync_local_race_conflicted(self):
         self.db1 = self.create_database('test1', 'source')
@@ -1075,7 +991,7 @@ class SoledadBackendSyncTests(
         doc = self.db1.create_doc_from_json(simple_doc)
         doc_id = doc.doc_id
         doc1_rev = doc.rev
-        self.db1.create_index('test-idx', 'key')
+        # self.db1.create_index('test-idx', 'key')
         self.sync(self.db1, self.db2)
         content1 = '{"key": "localval"}'
         content2 = '{"key": "altval"}'
@@ -1094,21 +1010,22 @@ class SoledadBackendSyncTests(
         self.sync(self.db1, self.db2, trace_hook=after_whatschanged)
         self.assertEqual([True], triggered)
         self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True)
-        from_idx = self.db1.get_from_index('test-idx', 'altval')[0]
-        self.assertEqual(doc.doc_id, from_idx.doc_id)
-        self.assertEqual(doc.rev, from_idx.rev)
-        self.assertTrue(from_idx.has_conflicts)
-        self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
-        self.assertEqual([], self.db1.get_from_index('test-idx', 'localval'))
+        # soledad doesnt support indexing due to encryption
+        # from_idx = self.db1.get_from_index('test-idx', 'altval')[0]
+        # self.assertEqual(doc.doc_id, from_idx.doc_id)
+        # self.assertEqual(doc.rev, from_idx.rev)
+        # self.assertTrue(from_idx.has_conflicts)
+        # self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
+        # self.assertEqual([], self.db1.get_from_index('test-idx', 'localval'))
 
     def test_sync_propagates_deletes(self):
         self.db1 = self.create_database('test1', 'source')
         self.db2 = self.create_database('test2', 'both')
         doc1 = self.db1.create_doc_from_json(simple_doc)
         doc_id = doc1.doc_id
-        self.db1.create_index('test-idx', 'key')
+        # self.db1.create_index('test-idx', 'key')
         self.sync(self.db1, self.db2)
-        self.db2.create_index('test-idx', 'key')
+        # self.db2.create_index('test-idx', 'key')
         self.db3 = self.create_database('test3', 'target')
         self.sync(self.db1, self.db3)
         self.db1.delete_doc(doc1)
@@ -1124,8 +1041,8 @@ class SoledadBackendSyncTests(
             self.db1, doc_id, deleted_rev, None, False)
         self.assertGetDocIncludeDeleted(
             self.db2, doc_id, deleted_rev, None, False)
-        self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
-        self.assertEqual([], self.db2.get_from_index('test-idx', 'value'))
+        # self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
+        # self.assertEqual([], self.db2.get_from_index('test-idx', 'value'))
         self.sync(self.db2, self.db3)
         self.assertLastExchangeLog(
             self.db3,
@@ -1324,9 +1241,11 @@ class SoledadBackendExceptionsTests(CouchDBTestCase):
     def create_db(self, ensure=True, dbname=None):
         if not dbname:
             dbname = ('test-%s' % uuid4().hex)
-        self.db = couch.CouchDatabase.open_database(
-            urljoin('http://127.0.0.1:%d' % self.couch_port, dbname),
-            create=True,
+        if dbname not in self.couch_server:
+            self.couch_server.create(dbname)
+        self.db = couch.CouchDatabase(
+            ('http://127.0.0.1:%d' % self.couch_port),
+            dbname,
             ensure_ddocs=ensure)
 
     def tearDown(self):
@@ -1520,14 +1439,14 @@ class SoledadBackendExceptionsTests(CouchDBTestCase):
 class DatabaseNameValidationTest(unittest.TestCase):
 
     def test_database_name_validation(self):
-        self.assertFalse(couch.is_db_name_valid("user-deadbeef | cat /secret"))
-        self.assertTrue(couch.is_db_name_valid("user-cafe1337"))
+        self.assertFalse(couch.state.is_db_name_valid("user-deadbeef | cat /secret"))
+        self.assertTrue(couch.state.is_db_name_valid("user-cafe1337"))
 
 
 class CommandBasedDBCreationTest(unittest.TestCase):
 
     def test_ensure_db_using_custom_command(self):
-        state = couch.CouchServerState("url", create_cmd="echo")
+        state = couch.state.CouchServerState("url", create_cmd="echo")
         mock_db = Mock()
         mock_db.replica_uid = 'replica_uid'
         state.open_database = Mock(return_value=mock_db)
@@ -1536,11 +1455,11 @@ class CommandBasedDBCreationTest(unittest.TestCase):
         self.assertEquals(mock_db.replica_uid, replica_uid)
 
     def test_raises_unauthorized_on_failure(self):
-        state = couch.CouchServerState("url", create_cmd="inexistent")
+        state = couch.state.CouchServerState("url", create_cmd="inexistent")
         self.assertRaises(u1db_errors.Unauthorized,
                           state.ensure_database, "user-1337")
 
     def test_raises_unauthorized_by_default(self):
-        state = couch.CouchServerState("url")
+        state = couch.state.CouchServerState("url")
         self.assertRaises(u1db_errors.Unauthorized,
                           state.ensure_database, "user-1337")
diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
index 2c2daf05..8cd3ae08 100644
--- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
+++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py
@@ -26,7 +26,8 @@ from twisted.internet import defer
 from uuid import uuid4
 
 from leap.soledad.client import Soledad
-from leap.soledad.common.couch import SoledadBackend, CouchServerState
+from leap.soledad.common.couch.state import CouchServerState
+from leap.soledad.common.couch import CouchDatabase
 
 from leap.soledad.common.tests.util import (
     make_token_soledad_app,
@@ -86,7 +87,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer):
         TestCaseWithServer.setUp(self)
         CouchDBTestCase.setUp(self)
         self.user = ('user-%s' % uuid4().hex)
-        self.db = SoledadBackend.open_database(
+        self.db = CouchDatabase.open_database(
             urljoin(self.couch_url, 'user-' + self.user),
             create=True,
             replica_uid='replica',
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index 5810ce31..d75275d6 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -29,10 +29,8 @@ from urlparse import urljoin
 from twisted.internet import defer
 from twisted.trial import unittest
 
-from leap.soledad.common.couch import (
-    CouchServerState,
-    SoledadBackend,
-)
+from leap.soledad.common.couch.state import CouchServerState
+from leap.soledad.common.couch import CouchDatabase
 from leap.soledad.common.tests.u1db_tests import TestCaseWithServer
 from leap.soledad.common.tests.test_couch import CouchDBTestCase
 from leap.soledad.common.tests.util import (
@@ -358,7 +356,7 @@ class EncryptedSyncTestCase(
             passphrase=passphrase)
 
         # ensure remote db exists before syncing
-        db = SoledadBackend.open_database(
+        db = CouchDatabase.open_database(
             urljoin(self.couch_url, 'user-' + user),
             create=True,
             ensure_ddocs=True)
@@ -488,7 +486,7 @@ class LockResourceTestCase(
         self.tempdir = tempfile.mkdtemp(prefix="leap_tests-")
         TestCaseWithServer.setUp(self)
         # create the databases
-        db = SoledadBackend.open_database(
+        db = CouchDatabase.open_database(
             urljoin(self.couch_url, ('shared-%s' % (uuid4().hex))),
             create=True,
             ensure_ddocs=True)
diff --git a/common/src/leap/soledad/common/tests/test_sync_mutex.py b/common/src/leap/soledad/common/tests/test_sync_mutex.py
index 35cecb8f..973a8587 100644
--- a/common/src/leap/soledad/common/tests/test_sync_mutex.py
+++ b/common/src/leap/soledad/common/tests/test_sync_mutex.py
@@ -33,7 +33,8 @@ from twisted.internet import defer
 
 from leap.soledad.client.sync import SoledadSynchronizer
 
-from leap.soledad.common import couch
+from leap.soledad.common.couch.state import CouchServerState
+from leap.soledad.common.couch import CouchDatabase
 from leap.soledad.common.tests.u1db_tests import TestCaseWithServer
 from leap.soledad.common.tests.test_couch import CouchDBTestCase
 
@@ -84,7 +85,7 @@ class TestSyncMutex(
     sync_target = soledad_sync_target
 
     def make_app(self):
-        self.request_state = couch.CouchServerState(self.couch_url)
+        self.request_state = CouchServerState(self.couch_url)
         return self.make_app_with_state(self.request_state)
 
     def setUp(self):
@@ -102,7 +103,7 @@ class TestSyncMutex(
         self.startServer()
 
         # ensure remote db exists before syncing
-        db = couch.SoledadBackend.open_database(
+        db = CouchDatabase.open_database(
             urljoin(self.couch_url, 'user-' + self.user),
             create=True,
             ensure_ddocs=True)
diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py
index 0cf31c3f..f25e84dd 100644
--- a/common/src/leap/soledad/common/tests/test_sync_target.py
+++ b/common/src/leap/soledad/common/tests/test_sync_target.py
@@ -29,7 +29,6 @@ from uuid import uuid4
 
 from testscenarios import TestWithScenarios
 from twisted.internet import defer
-from urlparse import urljoin
 
 from leap.soledad.client import http_target as target
 from leap.soledad.client import crypto
@@ -37,7 +36,6 @@ from leap.soledad.client.sqlcipher import SQLCipherU1DBSync
 from leap.soledad.client.sqlcipher import SQLCipherOptions
 from leap.soledad.client.sqlcipher import SQLCipherDatabase
 
-from leap.soledad.common import couch
 from leap.soledad.common.document import SoledadDocument
 
 from leap.soledad.common.tests import u1db_tests as tests
@@ -265,9 +263,9 @@ class TestSoledadSyncTarget(
                                      replica_trans_id=replica_trans_id,
                                      number_of_docs=number_of_docs,
                                      doc_idx=doc_idx, sync_id=sync_id)
-        from leap.soledad.common.tests.test_couch import IndexedSoledadBackend
+        from leap.soledad.common.backend import SoledadBackend
         self.patch(
-            IndexedSoledadBackend, '_put_doc_if_newer', bomb_put_doc_if_newer)
+            SoledadBackend, '_put_doc_if_newer', bomb_put_doc_if_newer)
         remote_target = self.getSyncTarget(
             source_replica_uid='replica')
         other_changes = []
diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py
index bfd06856..f7f9ebd0 100644
--- a/common/src/leap/soledad/common/tests/util.py
+++ b/common/src/leap/soledad/common/tests/util.py
@@ -27,7 +27,6 @@ import shutil
 import random
 import string
 import u1db
-import traceback
 import couchdb
 
 from uuid import uuid4
@@ -37,17 +36,17 @@ from StringIO import StringIO
 from pysqlcipher import dbapi2
 
 from u1db import sync
-from u1db.errors import DatabaseDoesNotExist
 from u1db.remote import http_database
 
 from twisted.trial import unittest
 
-from leap.common.files import mkdir_p
 from leap.common.testing.basetest import BaseLeapTest
 
 from leap.soledad.common import soledad_assert
 from leap.soledad.common.document import SoledadDocument
-from leap.soledad.common.couch import SoledadBackend, CouchServerState
+from leap.soledad.common.couch import CouchDatabase
+from leap.soledad.common.couch.state import CouchServerState
+
 from leap.soledad.common.crypto import ENC_SCHEME_KEY
 
 from leap.soledad.client import Soledad
@@ -379,7 +378,7 @@ class CouchServerStateForTests(CouchServerState):
         Create db and append to a list, allowing test to close it later
         """
         dbname = dbname or ('test-%s' % uuid4().hex)
-        db = SoledadBackend.open_database(
+        db = CouchDatabase.open_database(
             urljoin(self.couch_url, dbname),
             True,
             replica_uid=replica_uid or 'test',
diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db
index a8ba3833..54856643 100755
--- a/server/pkg/create-user-db
+++ b/server/pkg/create-user-db
@@ -19,8 +19,8 @@ import os
 import sys
 import netrc
 import argparse
-from leap.soledad.common.couch import SoledadBackend
-from leap.soledad.common.couch import is_db_name_valid
+from leap.soledad.common.couch import CouchDatabase
+from leap.soledad.common.couch.state import is_db_name_valid
 from leap.soledad.common.couch import list_users_dbs
 from leap.soledad.server import load_configuration
 
@@ -69,7 +69,7 @@ def ensure_database(dbname):
         sys.exit(1)
     url = url_for_db(dbname)
     db_security = CONF['database-security']
-    db = SoledadBackend.open_database(url=url, create=True,
+    db = CouchDatabase.open_database(url=url, create=True,
                                      replica_uid=None, ensure_ddocs=True,
                                      database_security=db_security)
     print ('success! Ensured that database %s exists, with replica_uid: %s' %
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 618ccb2b..00e1e9fb 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -104,7 +104,7 @@ from leap.soledad.server.sync import (
 )
 
 from leap.soledad.common import SHARED_DB_NAME
-from leap.soledad.common.couch import CouchServerState
+from leap.soledad.common.couch.state import CouchServerState
 
 # ----------------------------------------------------------------------------
 # Soledad WSGI application
-- 
cgit v1.2.3


From 9830bd00964a1c83cfc92151446adda6f14fbc54 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 20 Oct 2015 15:31:09 -0300
Subject: [refactor] CouchDocument is now ServerDocument

As SoledadBackend is intended to be database agnostic, a new generic
document is now used instead of the old one made for CouchDB. The only
attribute that really relates to couch was couch_rev, removed on this
commit as it can be set on CouchDatabase implementation when needed.
---
 common/src/leap/soledad/common/backend.py          | 43 ++++++++++------------
 common/src/leap/soledad/common/couch/__init__.py   | 24 ++++++------
 common/src/leap/soledad/common/document.py         | 25 ++++---------
 common/src/leap/soledad/common/tests/test_couch.py |  6 +--
 scripts/profiling/mail/mx.py                       |  5 ++-
 5 files changed, 44 insertions(+), 59 deletions(-)

diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
index cb37b4ac..1cbf3d57 100644
--- a/common/src/leap/soledad/common/backend.py
+++ b/common/src/leap/soledad/common/backend.py
@@ -29,7 +29,7 @@ from u1db.errors import (
 )
 from u1db.backends import CommonBackend
 from u1db.backends import CommonSyncTarget
-from leap.soledad.common.document import CouchDocument
+from leap.soledad.common.document import ServerDocument
 
 
 class SoledadBackend(CommonBackend):
@@ -48,7 +48,7 @@ class SoledadBackend(CommonBackend):
         :type replica_uid: str
         """
         # save params
-        self._factory = CouchDocument
+        self._factory = ServerDocument
         self._real_replica_uid = None
         self._cache = None
         self._dbname = database._dbname
@@ -83,7 +83,7 @@ class SoledadBackend(CommonBackend):
 
     def delete_database(self):
         """
-        Delete a U1DB CouchDB database.
+        Delete a U1DB database.
         """
         self._database.delete_database()
 
@@ -221,7 +221,7 @@ class SoledadBackend(CommonBackend):
         :type check_for_conflicts: bool
 
         :return: The document.
-        :rtype: CouchDocument
+        :rtype: ServerDocument
         """
         return self._database._get_doc(doc_id, check_for_conflicts)
 
@@ -237,7 +237,7 @@ class SoledadBackend(CommonBackend):
         :type include_deleted: bool
 
         :return: A document object.
-        :rtype: CouchDocument.
+        :rtype: ServerDocument.
         """
         doc = self._get_doc(doc_id, check_for_conflicts=True)
         if doc is None:
@@ -255,25 +255,25 @@ class SoledadBackend(CommonBackend):
                                 documents will not be included in the results.
         :type include_deleted: bool
 
-        :return: (generation, [CouchDocument])
+        :return: (generation, [ServerDocument])
             The current generation of the database, followed by a list of all
             the documents in the database.
-        :rtype: (int, [CouchDocument])
+        :rtype: (int, [ServerDocument])
         """
         return self._database.get_all_docs(include_deleted)
 
     def _put_doc(self, old_doc, doc):
         """
-        Put the document in the Couch backend database.
+        Put the document in the backend database.
 
         Note that C{old_doc} must have been fetched with the parameter
         C{check_for_conflicts} equal to True, so we can properly update the
         new document using the conflict information from the old one.
 
         :param old_doc: The old document version.
-        :type old_doc: CouchDocument
+        :type old_doc: ServerDocument
         :param doc: The document to be put.
-        :type doc: CouchDocument
+        :type doc: ServerDocument
         """
         last_transaction =\
             self._database.save_document(old_doc, doc,
@@ -348,7 +348,7 @@ class SoledadBackend(CommonBackend):
         This will also set doc.content to None.
 
         :param doc: The document to mark as deleted.
-        :type doc: CouchDocument.
+        :type doc: ServerDocument.
 
         :raise DocumentDoesNotExist: Raised if the document does not
                                             exist.
@@ -372,20 +372,17 @@ class SoledadBackend(CommonBackend):
         self._put_doc(old_doc, doc)
         return new_rev
 
-    def get_doc_conflicts(self, doc_id, couch_rev=None):
+    def get_doc_conflicts(self, doc_id):
         """
         Get the conflicted versions of a document.
 
-        If the C{couch_rev} parameter is not None, conflicts for a specific
-        document's couch revision are returned.
-
-        :param couch_rev: The couch document revision.
-        :type couch_rev: str
+        :param doc_id: The document id.
+        :type doc_id: str
 
         :return: A list of conflicted versions of the document.
         :rtype: list
         """
-        return self._database.get_doc_conflicts(doc_id, couch_rev)
+        return self._database.get_doc_conflicts(doc_id)
 
     def _get_replica_gen_and_trans_id(self, other_replica_uid):
         """
@@ -478,7 +475,7 @@ class SoledadBackend(CommonBackend):
         Add a conflict and force a document put.
 
         :param doc: The document to be put.
-        :type doc: CouchDocument
+        :type doc: ServerDocument
         """
         my_doc = self._get_doc(doc.doc_id)
         self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev))
@@ -499,7 +496,7 @@ class SoledadBackend(CommonBackend):
         the time you GET_DOC_CONFLICTS until the point where you RESOLVE)
 
         :param doc: A Document with the new content to be inserted.
-        :type doc: CouchDocument
+        :type doc: ServerDocument
         :param conflicted_doc_revs: A list of revisions that the new content
                                     supersedes.
         :type conflicted_doc_revs: [str]
@@ -564,7 +561,7 @@ class SoledadBackend(CommonBackend):
         remote copy is never updated again.)
 
         :param doc: A document object
-        :type doc: CouchDocument
+        :type doc: ServerDocument
         :param save_conflict: If this document is a conflict, do you want to
                               save it as a conflict, or just ignore it.
         :type save_conflict: bool
@@ -596,7 +593,7 @@ class SoledadBackend(CommonBackend):
                  'converged', at_gen is the insertion/current generation.
         :rtype: (str, int)
         """
-        if not isinstance(doc, CouchDocument):
+        if not isinstance(doc, ServerDocument):
             doc = self._factory(doc.doc_id, doc.rev, doc.get_json())
         my_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
         if my_doc:
@@ -636,7 +633,7 @@ class SoledadBackend(CommonBackend):
         Originally in u1db.CommonBackend
 
         :param doc: The document to have conflicts pruned.
-        :type doc: CouchDocument
+        :type doc: ServerDocument
         :param doc_vcr: A vector clock representing the current document's
                         revision.
         :type doc_vcr: u1db.vectorclock.VectorClock
diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
index 727f033f..1780cecd 100644
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# couch.py
+# __init__.py
 # Copyright (C) 2013 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
@@ -54,7 +54,7 @@ from leap.soledad.common import ddocs
 from leap.soledad.common.errors import raise_server_error
 from leap.soledad.common.errors import raise_missing_design_doc_error
 from leap.soledad.common.errors import InvalidURLError
-from leap.soledad.common.document import CouchDocument
+from leap.soledad.common.document import ServerDocument
 from leap.soledad.common.backend import SoledadBackend
 
 
@@ -78,8 +78,8 @@ def list_users_dbs(couch_url):
     return users
 
 
-# monkey-patch the u1db http app to use CouchDocument
-http_app.Document = CouchDocument
+# monkey-patch the u1db http app to use ServerDocument
+http_app.Document = ServerDocument
 
 
 @contextmanager
@@ -254,10 +254,10 @@ class CouchDatabase(object):
                                 documents will not be included in the results.
         :type include_deleted: bool
 
-        :return: (generation, [CouchDocument])
+        :return: (generation, [ServerDocument])
             The current generation of the database, followed by a list of all
             the documents in the database.
-        :rtype: (int, [CouchDocument])
+        :rtype: (int, [ServerDocument])
         """
 
         generation, _ = self._get_generation_info()
@@ -316,7 +316,7 @@ class CouchDatabase(object):
         :type check_for_conflicts: bool
 
         :return: The document.
-        :rtype: CouchDocument
+        :rtype: ServerDocument
         """
         # get document with all attachments (u1db content and eventual
         # conflicts)
@@ -333,7 +333,7 @@ class CouchDatabase(object):
         # restrict to u1db documents
         if 'u1db_rev' not in result:
             return None
-        doc = CouchDocument(doc_id, result['u1db_rev'])
+        doc = ServerDocument(doc_id, result['u1db_rev'])
         # set contents or make tombstone
         if '_attachments' not in result \
                 or 'u1db_content' not in result['_attachments']:
@@ -368,7 +368,7 @@ class CouchDatabase(object):
         """
         conflicts = []
         for doc_rev, content in attached_conflicts:
-            doc = CouchDocument(doc_id, doc_rev)
+            doc = ServerDocument(doc_id, doc_rev)
             if content is None:
                 doc.make_tombstone()
             else:
@@ -652,9 +652,9 @@ class CouchDatabase(object):
         new document using the conflict information from the old one.
 
         :param old_doc: The old document version.
-        :type old_doc: CouchDocument
+        :type old_doc: ServerDocument
         :param doc: The document to be put.
-        :type doc: CouchDocument
+        :type doc: ServerDocument
 
         :raise RevisionConflict: Raised when trying to update a document but
                                  couch revisions mismatch.
@@ -712,7 +712,7 @@ class CouchDatabase(object):
             '_attachments': attachments,
         }
         # if we are updating a doc we have to add the couch doc revision
-        if old_doc is not None:
+        if old_doc is not None and hasattr(old_doc, 'couch_rev'):
             couch_doc['_rev'] = old_doc.couch_rev
         # prepare the multipart PUT
         buf = StringIO()
diff --git a/common/src/leap/soledad/common/document.py b/common/src/leap/soledad/common/document.py
index 05458906..20510544 100644
--- a/common/src/leap/soledad/common/document.py
+++ b/common/src/leap/soledad/common/document.py
@@ -110,18 +110,17 @@ class SoledadDocument(Document):
         doc="Wrapper to ensure `doc.rev` is always returned as bytes.")
 
 
-class CouchDocument(SoledadDocument):
+class ServerDocument(SoledadDocument):
     """
-    This is the document used for maintaining the Couch backend.
+    This is the document used by server to hold conflicts and transactions
+    on a database.
 
-    A CouchDocument can fetch and manipulate conflicts and also holds a
-    reference to the couch document revision. This data is used to ensure an
-    atomic and consistent update of the database.
+    The goal is to ensure an atomic and consistent update of the database.
     """
 
     def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False):
         """
-        Container for handling a document that is stored in couch backend.
+        Container for handling a document that stored on server.
 
         :param doc_id: The unique document identifier.
         :type doc_id: str
@@ -133,7 +132,6 @@ class CouchDocument(SoledadDocument):
         :type has_conflicts: bool
         """
         SoledadDocument.__init__(self, doc_id, rev, json, has_conflicts)
-        self.couch_rev = None
         self.transactions = None
         self._conflicts = None
 
@@ -142,7 +140,7 @@ class CouchDocument(SoledadDocument):
         Get the conflicted versions of the document.
 
         :return: The conflicted versions of the document.
-        :rtype: [CouchDocument]
+        :rtype: [ServerDocument]
         """
         return self._conflicts or []
 
@@ -161,7 +159,7 @@ class CouchDocument(SoledadDocument):
         Add a conflict to this document.
 
         :param doc: The conflicted version to be added.
-        :type doc: CouchDocument
+        :type doc: Document
         """
         if self._conflicts is None:
             raise Exception("Fetch conflicts first!")
@@ -181,12 +179,3 @@ class CouchDocument(SoledadDocument):
             lambda doc: doc.rev not in conflict_revs,
             self._conflicts)
         self.has_conflicts = len(self._conflicts) > 0
-
-    def update(self, new_doc):
-        # update info
-        self.rev = new_doc.rev
-        if new_doc.is_tombstone():
-            self.is_tombstone()
-        else:
-            self.content = new_doc.content
-        self.has_conflicts = new_doc.has_conflicts
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 1634d30c..01f0587b 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -36,7 +36,7 @@ from u1db import SyncTarget
 from u1db import vectorclock
 
 from leap.soledad.common import couch
-from leap.soledad.common import backend
+from leap.soledad.common.document import ServerDocument
 from leap.soledad.common import errors
 
 from leap.soledad.common.tests import u1db_tests as tests
@@ -47,8 +47,6 @@ from leap.soledad.common.tests.util import sync_via_synchronizer
 from leap.soledad.common.tests.u1db_tests import test_backends
 from leap.soledad.common.tests.u1db_tests import DatabaseBaseTests
 
-from u1db.backends.inmemory import InMemoryIndex
-
 
 # -----------------------------------------------------------------------------
 # The following tests come from `u1db.tests.test_common_backend`.
@@ -134,7 +132,7 @@ def copy_couch_database_for_test(test, db):
 
 
 def make_document_for_test(test, doc_id, rev, content, has_conflicts=False):
-    return couch.CouchDocument(
+    return ServerDocument(
         doc_id, rev, content, has_conflicts=has_conflicts)
 
 
diff --git a/scripts/profiling/mail/mx.py b/scripts/profiling/mail/mx.py
index b6a1e5cf..55084a40 100644
--- a/scripts/profiling/mail/mx.py
+++ b/scripts/profiling/mail/mx.py
@@ -5,7 +5,7 @@ import timeit
 
 
 from leap.keymanager import openpgp
-from leap.soledad.common.couch import CouchDocument
+from leap.soledad.common.document import ServerDocument
 from leap.soledad.common.crypto import (
     EncryptionSchemes,
     ENC_JSON_KEY,
@@ -47,7 +47,7 @@ def get_enc_json(pubkey, message):
 
 
 def get_new_doc(enc_json):
-    doc = CouchDocument(doc_id=str(uuid.uuid4()))
+    doc = ServerDocument(doc_id=str(uuid.uuid4()))
     doc.content = {
         'incoming': True,
         ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY,
@@ -71,6 +71,7 @@ def put_lots_of_messages(db, number):
     log("Populating database with %d encrypted messages... "
         % number, line_break=False)
     pubkey = get_pubkey()
+
     def _put_one_message():
         put_one_message(pubkey, db)
     time = timeit.timeit(_put_one_message, number=number)
-- 
cgit v1.2.3


From b9de2af55ab7fbccadf1fdb92d06858c9f29acfa Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 20 Oct 2015 16:50:17 -0300
Subject: [refactor] move couch errors and expose methods

errors.py was holding a few specific CouchDB errors, now moved into
couch.errors module. Also, some of CouchDatabase methods were declared
as private, but external classes needs them.
---
 client/src/leap/soledad/client/sync.py             |   2 +-
 common/src/leap/soledad/common/backend.py          |  99 +++-----------
 common/src/leap/soledad/common/couch/__init__.py   |  36 ++----
 common/src/leap/soledad/common/couch/errors.py     | 144 +++++++++++++++++++++
 common/src/leap/soledad/common/errors.py           | 124 ------------------
 common/src/leap/soledad/common/tests/test_couch.py |  52 ++++----
 6 files changed, 206 insertions(+), 251 deletions(-)
 create mode 100644 common/src/leap/soledad/common/couch/errors.py

diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py
index 2276db2a..626ad2e5 100644
--- a/client/src/leap/soledad/client/sync.py
+++ b/client/src/leap/soledad/client/sync.py
@@ -22,7 +22,7 @@ import logging
 from twisted.internet import defer
 
 from u1db import errors
-from leap.soledad.common.errors import MissingDesignDocError
+from leap.soledad.common.couch.errors import MissingDesignDocError
 from u1db.sync import Synchronizer
 
 
diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
index 1cbf3d57..2f0ef4f7 100644
--- a/common/src/leap/soledad/common/backend.py
+++ b/common/src/leap/soledad/common/backend.py
@@ -141,19 +141,7 @@ class SoledadBackend(CommonBackend):
         :return: The current generation.
         :rtype: int
 
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
+        :raise SoledadError: Raised by database on operation failure
         """
         return self._get_generation_info()[0]
 
@@ -164,24 +152,12 @@ class SoledadBackend(CommonBackend):
         :return: A tuple containing the current generation and transaction id.
         :rtype: (int, str)
 
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
+        :raise SoledadError: Raised by database on operation failure
         """
         if self.replica_uid + '_gen' in self.cache:
             response = self.cache[self.replica_uid + '_gen']
             return response
-        cur_gen, newest_trans_id = self._database._get_generation_info()
+        cur_gen, newest_trans_id = self._database.get_generation_info()
         self.cache[self.replica_uid + '_gen'] = (cur_gen, newest_trans_id)
         return (cur_gen, newest_trans_id)
 
@@ -195,8 +171,10 @@ class SoledadBackend(CommonBackend):
         :return: The transaction id for C{generation}.
         :rtype: str
 
+        :raise InvalidGeneration: Raised when the generation does not exist.
+
         """
-        return self._database._get_trans_id_for_gen(generation)
+        return self._database.get_trans_id_for_gen(generation)
 
     def _get_transaction_log(self):
         """
@@ -206,7 +184,7 @@ class SoledadBackend(CommonBackend):
         :rtype: [(str, str)]
 
         """
-        return self._database._get_transaction_log()
+        return self._database.get_transaction_log()
 
     def _get_doc(self, doc_id, check_for_conflicts=False):
         """
@@ -223,7 +201,7 @@ class SoledadBackend(CommonBackend):
         :return: The document.
         :rtype: ServerDocument
         """
-        return self._database._get_doc(doc_id, check_for_conflicts)
+        return self._database.get_doc(doc_id, check_for_conflicts)
 
     def get_doc(self, doc_id, include_deleted=False):
         """
@@ -403,12 +381,10 @@ class SoledadBackend(CommonBackend):
                  synchronized with the replica, this is (0, '').
         :rtype: (int, str)
         """
-        return self._database._get_replica_gen_and_trans_id(other_replica_uid)
+        return self._database.get_replica_gen_and_trans_id(other_replica_uid)
 
     def _set_replica_gen_and_trans_id(self, other_replica_uid,
-                                      other_generation, other_transaction_id,
-                                      number_of_docs=None, doc_idx=None,
-                                      sync_id=None):
+                                      other_generation, other_transaction_id):
         """
         Set the last-known generation and transaction id for the other
         database replica.
@@ -424,30 +400,18 @@ class SoledadBackend(CommonBackend):
         :param other_transaction_id: The transaction id associated with the
             generation.
         :type other_transaction_id: str
-        :param number_of_docs: The total amount of documents sent on this sync
-                               session.
-        :type number_of_docs: int
-        :param doc_idx: The index of the current document being sent.
-        :type doc_idx: int
-        :param sync_id: The id of the current sync session.
-        :type sync_id: str
         """
         if other_replica_uid is not None and other_generation is not None:
-            self._do_set_replica_gen_and_trans_id(
-                other_replica_uid, other_generation, other_transaction_id,
-                number_of_docs=number_of_docs, doc_idx=doc_idx,
-                sync_id=sync_id)
+            self.cache[other_replica_uid] = (other_generation,
+                                             other_transaction_id)
+            self._database.set_replica_gen_and_trans_id(other_replica_uid,
+                                                        other_generation,
+                                                        other_transaction_id)
 
     def _do_set_replica_gen_and_trans_id(
-            self, other_replica_uid, other_generation, other_transaction_id,
-            number_of_docs=None, doc_idx=None, sync_id=None):
+            self, other_replica_uid, other_generation, other_transaction_id):
         """
-        Set the last-known generation and transaction id for the other
-        database replica.
-
-        We have just performed some synchronization, and we want to track what
-        generation the other replica was at. See also
-        _get_replica_gen_and_trans_id.
+        _put_doc_if_newer from super class is calling it. So we declare this.
 
         :param other_replica_uid: The U1DB identifier for the other replica.
         :type other_replica_uid: str
@@ -456,19 +420,10 @@ class SoledadBackend(CommonBackend):
         :param other_transaction_id: The transaction id associated with the
                                      generation.
         :type other_transaction_id: str
-        :param number_of_docs: The total amount of documents sent on this sync
-                               session.
-        :type number_of_docs: int
-        :param doc_idx: The index of the current document being sent.
-        :type doc_idx: int
-        :param sync_id: The id of the current sync session.
-        :type sync_id: str
         """
-        self.cache[other_replica_uid] = (other_generation,
-                                         other_transaction_id)
-        self._database._do_set_replica_gen_and_trans_id(other_replica_uid,
-                                                        other_generation,
-                                                        other_transaction_id)
+        self._set_replica_gen_and_trans_id(other_replica_uid,
+                                           other_generation,
+                                           other_transaction_id)
 
     def _force_doc_sync_conflict(self, doc):
         """
@@ -501,19 +456,7 @@ class SoledadBackend(CommonBackend):
                                     supersedes.
         :type conflicted_doc_revs: [str]
 
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
+        :raise SoledadError: Raised by database on operation failure
         """
         cur_doc = self._get_doc(doc.doc_id, check_for_conflicts=True)
         new_rev = self._ensure_maximal_rev(cur_doc.rev,
diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
index 1780cecd..6fd4d0ce 100644
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -51,8 +51,8 @@ from u1db.remote import http_app
 
 
 from leap.soledad.common import ddocs
-from leap.soledad.common.errors import raise_server_error
-from leap.soledad.common.errors import raise_missing_design_doc_error
+from .errors import raise_server_error
+from .errors import raise_missing_design_doc_error
 from leap.soledad.common.errors import InvalidURLError
 from leap.soledad.common.document import ServerDocument
 from leap.soledad.common.backend import SoledadBackend
@@ -260,7 +260,7 @@ class CouchDatabase(object):
         :rtype: (int, [ServerDocument])
         """
 
-        generation, _ = self._get_generation_info()
+        generation, _ = self.get_generation_info()
         results = list(self.get_docs(self._database,
                                      include_deleted=include_deleted))
         return (generation, results)
@@ -294,7 +294,7 @@ class CouchDatabase(object):
         # This will not be needed when/if we switch from python-couchdb to
         # paisley.
         time.strptime('Mar 8 1917', '%b %d %Y')
-        get_one = lambda doc_id: self._get_doc(doc_id, check_for_conflicts)
+        get_one = lambda doc_id: self.get_doc(doc_id, check_for_conflicts)
         docs = [THREAD_POOL.apply_async(get_one, [doc_id])
                 for doc_id in doc_ids]
         for doc in docs:
@@ -303,7 +303,7 @@ class CouchDatabase(object):
                 continue
             yield doc
 
-    def _get_doc(self, doc_id, check_for_conflicts=False):
+    def get_doc(self, doc_id, check_for_conflicts=False):
         """
         Extract the document from storage.
 
@@ -376,7 +376,7 @@ class CouchDatabase(object):
             conflicts.append(doc)
         return conflicts
 
-    def _get_trans_id_for_gen(self, generation):
+    def get_trans_id_for_gen(self, generation):
         """
         Get the transaction id corresponding to a particular generation.
 
@@ -418,7 +418,7 @@ class CouchDatabase(object):
         except ServerError as e:
             raise_server_error(e, ddoc_path)
 
-    def _get_replica_gen_and_trans_id(self, other_replica_uid):
+    def get_replica_gen_and_trans_id(self, other_replica_uid):
         """
         Return the last known generation and transaction id for the other db
         replica.
@@ -470,7 +470,7 @@ class CouchDatabase(object):
             params['rev'] = couch_rev  # restric document's couch revision
         else:
             # TODO: move into resource logic!
-            first_entry = self._get_doc(doc_id, check_for_conflicts=True)
+            first_entry = self.get_doc(doc_id, check_for_conflicts=True)
             conflicts.append(first_entry)
         resource = self._database.resource(doc_id, 'u1db_conflicts')
         try:
@@ -480,16 +480,15 @@ class CouchDatabase(object):
         except ResourceNotFound:
             return []
 
-    def _do_set_replica_gen_and_trans_id(
-            self, other_replica_uid, other_generation, other_transaction_id,
-            number_of_docs=None, doc_idx=None, sync_id=None):
+    def set_replica_gen_and_trans_id(
+            self, other_replica_uid, other_generation, other_transaction_id):
         """
         Set the last-known generation and transaction id for the other
         database replica.
 
         We have just performed some synchronization, and we want to track what
         generation the other replica was at. See also
-        _get_replica_gen_and_trans_id.
+        get_replica_gen_and_trans_id.
 
         :param other_replica_uid: The U1DB identifier for the other replica.
         :type other_replica_uid: str
@@ -498,13 +497,6 @@ class CouchDatabase(object):
         :param other_transaction_id: The transaction id associated with the
                                      generation.
         :type other_transaction_id: str
-        :param number_of_docs: The total amount of documents sent on this sync
-                               session.
-        :type number_of_docs: int
-        :param doc_idx: The index of the current document being sent.
-        :type doc_idx: int
-        :param sync_id: The id of the current sync session.
-        :type sync_id: str
         """
         doc_id = 'u1db_sync_%s' % other_replica_uid
         try:
@@ -515,7 +507,7 @@ class CouchDatabase(object):
         doc['transaction_id'] = other_transaction_id
         self._database.save(doc)
 
-    def _get_transaction_log(self):
+    def get_transaction_log(self):
         """
         This is only for the test suite, it is not part of the api.
 
@@ -603,7 +595,7 @@ class CouchDatabase(object):
                 newest_trans_id = changes[0][2]
                 changes.reverse()
             else:
-                cur_gen, newest_trans_id = self._get_generation_info()
+                cur_gen, newest_trans_id = self.get_generation_info()
 
             return cur_gen, newest_trans_id, changes
         except ResourceNotFound as e:
@@ -611,7 +603,7 @@ class CouchDatabase(object):
         except ServerError as e:
             raise_server_error(e, ddoc_path)
 
-    def _get_generation_info(self):
+    def get_generation_info(self):
         """
         Return the current generation.
 
diff --git a/common/src/leap/soledad/common/couch/errors.py b/common/src/leap/soledad/common/couch/errors.py
new file mode 100644
index 00000000..e894d58f
--- /dev/null
+++ b/common/src/leap/soledad/common/couch/errors.py
@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+# errors.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+from leap.soledad.common.errors import SoledadError
+from leap.soledad.common.errors import register_exception
+
+"""
+Specific errors that can be raised by CouchDatabase.
+"""
+
+
+@register_exception
+class MissingDesignDocError(SoledadError):
+
+    """
+    Raised when trying to access a missing couch design document.
+    """
+
+    wire_description = "missing design document"
+    status = 500
+
+
+@register_exception
+class MissingDesignDocNamedViewError(SoledadError):
+
+    """
+    Raised when trying to access a missing named view on a couch design
+    document.
+    """
+
+    wire_description = "missing design document named function"
+    status = 500
+
+
+@register_exception
+class MissingDesignDocListFunctionError(SoledadError):
+
+    """
+    Raised when trying to access a missing list function on a couch design
+    document.
+    """
+
+    wire_description = "missing design document list function"
+    status = 500
+
+
+@register_exception
+class MissingDesignDocDeletedError(SoledadError):
+
+    """
+    Raised when trying to access a deleted couch design document.
+    """
+
+    wire_description = "design document was deleted"
+    status = 500
+
+
+@register_exception
+class DesignDocUnknownError(SoledadError):
+
+    """
+    Raised when trying to access a couch design document and getting an
+    unknown error.
+    """
+
+    wire_description = "missing design document unknown error"
+    status = 500
+
+
+def raise_missing_design_doc_error(exc, ddoc_path):
+    """
+    Raise an appropriate exception when catching a ResourceNotFound when
+    accessing a design document.
+
+    :param exc: The exception cought.
+    :type exc: ResourceNotFound
+    :param ddoc_path: A list representing the requested path.
+    :type ddoc_path: list
+
+    :raise MissingDesignDocError: Raised when tried to access a missing design
+                                  document.
+    :raise MissingDesignDocListFunctionError: Raised when trying to access a
+                                              missing list function on a
+                                              design document.
+    :raise MissingDesignDocNamedViewError: Raised when trying to access a
+                                           missing named view on a design
+                                           document.
+    :raise MissingDesignDocDeletedError: Raised when trying to access a
+                                         deleted design document.
+    :raise MissingDesignDocUnknownError: Raised when failed to access a design
+                                         document for an yet unknown reason.
+    """
+    path = "".join(ddoc_path)
+    if exc.message[1] == 'missing':
+        raise MissingDesignDocError(path)
+    elif exc.message[1] == 'missing function' or \
+            exc.message[1].startswith('missing lists function'):
+        raise MissingDesignDocListFunctionError(path)
+    elif exc.message[1] == 'missing_named_view':
+        raise MissingDesignDocNamedViewError(path)
+    elif exc.message[1] == 'deleted':
+        raise MissingDesignDocDeletedError(path)
+    # other errors are unknown for now
+    raise DesignDocUnknownError("%s: %s" % (path, str(exc.message)))
+
+
+def raise_server_error(exc, ddoc_path):
+    """
+    Raise an appropriate exception when catching a ServerError when
+    accessing a design document.
+
+    :param exc: The exception cought.
+    :type exc: ResourceNotFound
+    :param ddoc_path: A list representing the requested path.
+    :type ddoc_path: list
+
+    :raise MissingDesignDocListFunctionError: Raised when trying to access a
+                                              missing list function on a
+                                              design document.
+    :raise MissingDesignDocUnknownError: Raised when failed to access a design
+                                         document for an yet unknown reason.
+    """
+    path = "".join(ddoc_path)
+    msg = exc.message[1][0]
+    if msg == 'unnamed_error':
+        raise MissingDesignDocListFunctionError(path)
+    elif msg == 'TypeError':
+        if 'point is undefined' in exc.message[1][1]:
+            raise MissingDesignDocListFunctionError
+    # other errors are unknown for now
+    raise DesignDocUnknownError("%s: %s" % (path, str(exc.message)))
diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py
index 84d8d813..2f6fd1d8 100644
--- a/common/src/leap/soledad/common/errors.py
+++ b/common/src/leap/soledad/common/errors.py
@@ -133,66 +133,6 @@ class CouldNotObtainLockError(SoledadError):
 
 #
 # SoledadBackend errors
-#
-
-@register_exception
-class MissingDesignDocError(SoledadError):
-
-    """
-    Raised when trying to access a missing couch design document.
-    """
-
-    wire_description = "missing design document"
-    status = 500
-
-
-@register_exception
-class MissingDesignDocNamedViewError(SoledadError):
-
-    """
-    Raised when trying to access a missing named view on a couch design
-    document.
-    """
-
-    wire_description = "missing design document named function"
-    status = 500
-
-
-@register_exception
-class MissingDesignDocListFunctionError(SoledadError):
-
-    """
-    Raised when trying to access a missing list function on a couch design
-    document.
-    """
-
-    wire_description = "missing design document list function"
-    status = 500
-
-
-@register_exception
-class MissingDesignDocDeletedError(SoledadError):
-
-    """
-    Raised when trying to access a deleted couch design document.
-    """
-
-    wire_description = "design document was deleted"
-    status = 500
-
-
-@register_exception
-class DesignDocUnknownError(SoledadError):
-
-    """
-    Raised when trying to access a couch design document and getting an
-    unknown error.
-    """
-
-    wire_description = "missing design document unknown error"
-    status = 500
-
-
 # u1db error statuses also have to be updated
 http_errors.ERROR_STATUSES = set(
     http_errors.wire_description_to_status.values())
@@ -203,67 +143,3 @@ class InvalidURLError(Exception):
     """
     Exception raised when Soledad encounters a malformed URL.
     """
-
-
-def raise_missing_design_doc_error(exc, ddoc_path):
-    """
-    Raise an appropriate exception when catching a ResourceNotFound when
-    accessing a design document.
-
-    :param exc: The exception cought.
-    :type exc: ResourceNotFound
-    :param ddoc_path: A list representing the requested path.
-    :type ddoc_path: list
-
-    :raise MissingDesignDocError: Raised when tried to access a missing design
-                                  document.
-    :raise MissingDesignDocListFunctionError: Raised when trying to access a
-                                              missing list function on a
-                                              design document.
-    :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                           missing named view on a design
-                                           document.
-    :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                         deleted design document.
-    :raise MissingDesignDocUnknownError: Raised when failed to access a design
-                                         document for an yet unknown reason.
-    """
-    path = "".join(ddoc_path)
-    if exc.message[1] == 'missing':
-        raise MissingDesignDocError(path)
-    elif exc.message[1] == 'missing function' or \
-            exc.message[1].startswith('missing lists function'):
-        raise MissingDesignDocListFunctionError(path)
-    elif exc.message[1] == 'missing_named_view':
-        raise MissingDesignDocNamedViewError(path)
-    elif exc.message[1] == 'deleted':
-        raise MissingDesignDocDeletedError(path)
-    # other errors are unknown for now
-    raise DesignDocUnknownError("%s: %s" % (path, str(exc.message)))
-
-
-def raise_server_error(exc, ddoc_path):
-    """
-    Raise an appropriate exception when catching a ServerError when
-    accessing a design document.
-
-    :param exc: The exception cought.
-    :type exc: ResourceNotFound
-    :param ddoc_path: A list representing the requested path.
-    :type ddoc_path: list
-
-    :raise MissingDesignDocListFunctionError: Raised when trying to access a
-                                              missing list function on a
-                                              design document.
-    :raise MissingDesignDocUnknownError: Raised when failed to access a design
-                                         document for an yet unknown reason.
-    """
-    path = "".join(ddoc_path)
-    msg = exc.message[1][0]
-    if msg == 'unnamed_error':
-        raise MissingDesignDocListFunctionError(path)
-    elif msg == 'TypeError':
-        if 'point is undefined' in exc.message[1][1]:
-            raise MissingDesignDocListFunctionError
-    # other errors are unknown for now
-    raise DesignDocUnknownError("%s: %s" % (path, str(exc.message)))
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index 01f0587b..a311e0ef 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -37,7 +37,7 @@ from u1db import vectorclock
 
 from leap.soledad.common import couch
 from leap.soledad.common.document import ServerDocument
-from leap.soledad.common import errors
+from leap.soledad.common.couch import errors
 
 from leap.soledad.common.tests import u1db_tests as tests
 from leap.soledad.common.tests.util import CouchDBTestCase
@@ -1257,18 +1257,18 @@ class SoledadBackendExceptionsTests(CouchDBTestCase):
         design docs are not present.
         """
         self.create_db(ensure=False)
-        # _get_generation_info()
+        # get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocError,
-            self.db._get_generation_info)
-        # _get_trans_id_for_gen()
+            self.db.get_generation_info)
+        # get_trans_id_for_gen()
         self.assertRaises(
             errors.MissingDesignDocError,
-            self.db._get_trans_id_for_gen, 1)
-        # _get_transaction_log()
+            self.db.get_trans_id_for_gen, 1)
+        # get_transaction_log()
         self.assertRaises(
             errors.MissingDesignDocError,
-            self.db._get_transaction_log)
+            self.db.get_transaction_log)
         # whats_changed()
         self.assertRaises(
             errors.MissingDesignDocError,
@@ -1284,14 +1284,14 @@ class SoledadBackendExceptionsTests(CouchDBTestCase):
         transactions = self.db._database['_design/transactions']
         transactions['lists'] = {}
         self.db._database.save(transactions)
-        # _get_generation_info()
+        # get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocListFunctionError,
-            self.db._get_generation_info)
-        # _get_trans_id_for_gen()
+            self.db.get_generation_info)
+        # get_trans_id_for_gen()
         self.assertRaises(
             errors.MissingDesignDocListFunctionError,
-            self.db._get_trans_id_for_gen, 1)
+            self.db.get_trans_id_for_gen, 1)
         # whats_changed()
         self.assertRaises(
             errors.MissingDesignDocListFunctionError,
@@ -1307,14 +1307,14 @@ class SoledadBackendExceptionsTests(CouchDBTestCase):
         transactions = self.db._database['_design/transactions']
         del transactions['lists']
         self.db._database.save(transactions)
-        # _get_generation_info()
+        # get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocListFunctionError,
-            self.db._get_generation_info)
+            self.db.get_generation_info)
         # _get_trans_id_for_gen()
         self.assertRaises(
             errors.MissingDesignDocListFunctionError,
-            self.db._get_trans_id_for_gen, 1)
+            self.db.get_trans_id_for_gen, 1)
         # whats_changed()
         self.assertRaises(
             errors.MissingDesignDocListFunctionError,
@@ -1338,18 +1338,18 @@ class SoledadBackendExceptionsTests(CouchDBTestCase):
         transactions = self.db._database['_design/transactions']
         del transactions['views']
         self.db._database.save(transactions)
-        # _get_generation_info()
+        # get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocNamedViewError,
-            self.db._get_generation_info)
+            self.db.get_generation_info)
         # _get_trans_id_for_gen()
         self.assertRaises(
             errors.MissingDesignDocNamedViewError,
-            self.db._get_trans_id_for_gen, 1)
+            self.db.get_trans_id_for_gen, 1)
         # _get_transaction_log()
         self.assertRaises(
             errors.MissingDesignDocNamedViewError,
-            self.db._get_transaction_log)
+            self.db.get_transaction_log)
         # whats_changed()
         self.assertRaises(
             errors.MissingDesignDocNamedViewError,
@@ -1367,18 +1367,18 @@ class SoledadBackendExceptionsTests(CouchDBTestCase):
         del self.db._database['_design/syncs']
         # delete _design/transactions
         del self.db._database['_design/transactions']
-        # _get_generation_info()
+        # get_generation_info()
         self.assertRaises(
             errors.MissingDesignDocDeletedError,
-            self.db._get_generation_info)
-        # _get_trans_id_for_gen()
+            self.db.get_generation_info)
+        # get_trans_id_for_gen()
         self.assertRaises(
             errors.MissingDesignDocDeletedError,
-            self.db._get_trans_id_for_gen, 1)
-        # _get_transaction_log()
+            self.db.get_trans_id_for_gen, 1)
+        # get_transaction_log()
         self.assertRaises(
             errors.MissingDesignDocDeletedError,
-            self.db._get_transaction_log)
+            self.db.get_transaction_log)
         # whats_changed()
         self.assertRaises(
             errors.MissingDesignDocDeletedError,
@@ -1393,9 +1393,9 @@ class SoledadBackendExceptionsTests(CouchDBTestCase):
         del self.db._database['_design/transactions']
         self.assertRaises(
             errors.MissingDesignDocDeletedError,
-            self.db._get_transaction_log)
+            self.db.get_transaction_log)
         self.create_db(ensure=True, dbname=self.db._dbname)
-        self.db._get_transaction_log()
+        self.db.get_transaction_log()
 
     def test_ensure_security_doc(self):
         """
-- 
cgit v1.2.3


From 421691ef71019d0bcd4447a773efa5e9b15b0c71 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 27 Oct 2015 16:48:39 -0300
Subject: [refactor] token verification moved to couch module + tests

Added tests for this token verification as it wasn't covered. Then moved
it to the new couch module that implements a couch storage.
The ServerState was chosen to hold the verify_token method.
CouchServerState holds the current implementation, which is called on
authentication middleware as the new test shows.
---
 common/src/leap/soledad/common/couch/state.py      | 46 +++++++++++++++++++
 .../src/leap/soledad/common/tests/test_server.py   | 31 +++++++++++++
 server/src/leap/soledad/server/auth.py             | 52 +---------------------
 3 files changed, 79 insertions(+), 50 deletions(-)

diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py
index 39d49fa0..40c0d55b 100644
--- a/common/src/leap/soledad/common/couch/state.py
+++ b/common/src/leap/soledad/common/couch/state.py
@@ -19,11 +19,14 @@ Server state using CouchDatabase as backend.
 """
 import re
 import logging
+import time
 from urlparse import urljoin
+from hashlib import sha512
 
 from u1db.remote.server_state import ServerState
 from leap.soledad.common.command import exec_validated_cmd
 from leap.soledad.common.couch import CouchDatabase
+from leap.soledad.common.couch import couch_server
 from u1db.errors import Unauthorized
 
 
@@ -50,6 +53,12 @@ class CouchServerState(ServerState):
     Inteface of the WSGI server with the CouchDB backend.
     """
 
+    TOKENS_DB_PREFIX = "tokens_"
+    TOKENS_DB_EXPIRE = 30 * 24 * 3600  # 30 days in seconds
+    TOKENS_TYPE_KEY = "type"
+    TOKENS_TYPE_DEF = "Token"
+    TOKENS_USER_ID_KEY = "user_id"
+
     def __init__(self, couch_url, create_cmd=None):
         """
         Initialize the couch server state.
@@ -112,3 +121,40 @@ class CouchServerState(ServerState):
                              delete databases.
         """
         raise Unauthorized()
+
+    def verify_token(self, uuid, token):
+        """
+        Query couchdb to decide if C{token} is valid for C{uuid}.
+
+        @param uuid: The user uuid.
+        @type uuid: str
+        @param token: The token.
+        @type token: str
+        """
+        with couch_server(self.couch_url) as server:
+            # the tokens db rotates every 30 days, and the current db name is
+            # "tokens_NNN", where NNN is the number of seconds since epoch divided
+            # by the rotate period in seconds. When rotating, old and new tokens
+            # db coexist during a certain window of time and valid tokens are
+            # replicated from the old db to the new one. See:
+            # https://leap.se/code/issues/6785
+            dbname = self._tokens_dbname()
+            db = server[dbname]
+        # lookup key is a hash of the token to prevent timing attacks.
+        token = db.get(sha512(token).hexdigest())
+        if token is None:
+            return False
+        # we compare uuid hashes to avoid possible timing attacks that
+        # might exploit python's builtin comparison operator behaviour,
+        # which fails immediatelly when non-matching bytes are found.
+        couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest()
+        req_uuid_hash = sha512(uuid).digest()
+        if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \
+                or couch_uuid_hash != req_uuid_hash:
+            return False
+        return True
+
+    def _tokens_dbname(self):
+        dbname = self.TOKENS_DB_PREFIX + \
+            str(int(time.time() / self.TOKENS_DB_EXPIRE))
+        return dbname
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index d75275d6..e1129a9f 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -24,6 +24,7 @@ import time
 import binascii
 from pkg_resources import resource_filename
 from uuid import uuid4
+from hashlib import sha512
 
 from urlparse import urljoin
 from twisted.internet import defer
@@ -46,6 +47,36 @@ from leap.soledad.server import LockResource
 from leap.soledad.server import load_configuration
 from leap.soledad.server import CONFIG_DEFAULTS
 from leap.soledad.server.auth import URLToAuthorization
+from leap.soledad.server.auth import SoledadTokenAuthMiddleware
+
+
+class ServerAuthenticationMiddlewareTestCase(CouchDBTestCase):
+
+    def setUp(self):
+        super(ServerAuthenticationMiddlewareTestCase, self).setUp()
+        app = mock.Mock()
+        self._state = CouchServerState(self.couch_url)
+        app.state = self._state
+        self.auth_middleware = SoledadTokenAuthMiddleware(app)
+        self._authorize('valid-uuid', 'valid-token')
+
+    def _authorize(self, uuid, token):
+        token_doc = {}
+        token_doc['_id'] = sha512(token).hexdigest()
+        token_doc[self._state.TOKENS_USER_ID_KEY] = uuid
+        token_doc[self._state.TOKENS_TYPE_KEY] = \
+            self._state.TOKENS_TYPE_DEF
+        dbname = self._state._tokens_dbname()
+        db = self.couch_server.create(dbname)
+        db.save(token_doc)
+        self.addCleanup(self.delete_db, db.name)
+
+    def test_authorized_user(self):
+        is_authorized = self.auth_middleware._verify_authentication_data
+        self.assertTrue(is_authorized('valid-uuid', 'valid-token'))
+        self.assertFalse(is_authorized('valid-uuid', 'invalid-token'))
+        self.assertFalse(is_authorized('invalid-uuid', 'valid-token'))
+        self.assertFalse(is_authorized('eve', 'invalid-token'))
 
 
 class ServerAuthorizationTestCase(BaseSoledadTest):
diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py
index 02b54cca..01baf1ce 100644
--- a/server/src/leap/soledad/server/auth.py
+++ b/server/src/leap/soledad/server/auth.py
@@ -21,20 +21,16 @@ Authentication facilities for Soledad Server.
 """
 
 
-import time
 import httplib
 import json
 
 from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors
 from abc import ABCMeta, abstractmethod
 from routes.mapper import Mapper
-from couchdb.client import Server
 from twisted.python import log
-from hashlib import sha512
 
 from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common import USER_DB_PREFIX
-from leap.soledad.common.errors import InvalidAuthTokenError
 
 
 class URLToAuthorization(object):
@@ -193,6 +189,7 @@ class SoledadAuthMiddleware(object):
         @type prefix: str
         """
         self._app = app
+        self._state = app.state
 
     def _error(self, start_response, status, description, message=None):
         """
@@ -351,12 +348,6 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
     Token based authentication.
     """
 
-    TOKENS_DB_PREFIX = "tokens_"
-    TOKENS_DB_EXPIRE = 30 * 24 * 3600  # 30 days in seconds
-    TOKENS_TYPE_KEY = "type"
-    TOKENS_TYPE_DEF = "Token"
-    TOKENS_USER_ID_KEY = "user_id"
-
     TOKEN_AUTH_ERROR_STRING = "Incorrect address or token."
 
     def _verify_authentication_scheme(self, scheme):
@@ -391,50 +382,11 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
         """
         token = auth_data  # we expect a cleartext token at this point
         try:
-            return self._verify_token_in_couch(uuid, token)
-        except InvalidAuthTokenError:
-            raise
+            return self._state.verify_token(uuid, token)
         except Exception as e:
             log.err(e)
             return False
 
-    def _verify_token_in_couch(self, uuid, token):
-        """
-        Query couchdb to decide if C{token} is valid for C{uuid}.
-
-        @param uuid: The user uuid.
-        @type uuid: str
-        @param token: The token.
-        @type token: str
-
-        @raise InvalidAuthTokenError: Raised when token received from user is
-                                      either missing in the tokens db or is
-                                      invalid.
-        """
-        server = Server(url=self._app.state.couch_url)
-        # the tokens db rotates every 30 days, and the current db name is
-        # "tokens_NNN", where NNN is the number of seconds since epoch divided
-        # by the rotate period in seconds. When rotating, old and new tokens
-        # db coexist during a certain window of time and valid tokens are
-        # replicated from the old db to the new one. See:
-        # https://leap.se/code/issues/6785
-        dbname = self.TOKENS_DB_PREFIX + \
-            str(int(time.time() / self.TOKENS_DB_EXPIRE))
-        db = server[dbname]
-        # lookup key is a hash of the token to prevent timing attacks.
-        token = db.get(sha512(token).hexdigest())
-        if token is None:
-            raise InvalidAuthTokenError()
-        # we compare uuid hashes to avoid possible timing attacks that
-        # might exploit python's builtin comparison operator behaviour,
-        # which fails immediatelly when non-matching bytes are found.
-        couch_uuid_hash = sha512(token[self.TOKENS_USER_ID_KEY]).digest()
-        req_uuid_hash = sha512(uuid).digest()
-        if token[self.TOKENS_TYPE_KEY] != self.TOKENS_TYPE_DEF \
-                or couch_uuid_hash != req_uuid_hash:
-            raise InvalidAuthTokenError()
-        return True
-
     def _get_auth_error_string(self):
         """
         Get the error string for token auth.
-- 
cgit v1.2.3


From eaf19626a57d5f5325653fee3aea9db27c9531fe Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 28 Oct 2015 17:58:27 -0300
Subject: [refactor] resource logic encapsulation

Creating a resource from a path to use get_json causes a lot of dirty
code and unexplained things like response[2]. This commit extracts that
logic into a helper to let it more clear about what is happening.
---
 common/src/leap/soledad/common/couch/__init__.py | 169 +++++++++--------------
 common/src/leap/soledad/common/document.py       |   1 -
 server/src/leap/soledad/server/auth.py           |   5 +-
 3 files changed, 71 insertions(+), 104 deletions(-)

diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
index 6fd4d0ce..274f59e2 100644
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -166,8 +166,8 @@ class CouchDatabase(object):
         """
         for ddoc_name in ['docs', 'syncs', 'transactions']:
             try:
-                self._database.resource('_design',
-                                        ddoc_name, '_info').get_json()
+                self.json_from_resource(['_design', ddoc_name, '_info'],
+                                        check_missing_ddoc=False)
             except ResourceNotFound:
                 ddoc = json.loads(
                     binascii.a2b_base64(
@@ -320,12 +320,9 @@ class CouchDatabase(object):
         """
         # get document with all attachments (u1db content and eventual
         # conflicts)
-        try:
-            result = \
-                self._database.resource(doc_id).get_json(
-                    attachments=True)[2]
-        except ResourceNotFound:
+        if doc_id not in self._database:
             return None
+        result = self.json_from_resource([doc_id], attachments=True)
         return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts)
 
     def __parse_doc_from_couch(self, result, doc_id,
@@ -387,19 +384,6 @@ class CouchDatabase(object):
         :rtype: str
 
         :raise InvalidGeneration: Raised when the generation does not exist.
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
         """
         if generation == 0:
             return ''
@@ -407,16 +391,10 @@ class CouchDatabase(object):
         ddoc_path = [
             '_design', 'transactions', '_list', 'trans_id_for_gen', 'log'
         ]
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json(gen=generation)
-            if response[2] == {}:
-                raise InvalidGeneration
-            return response[2]['transaction_id']
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-        except ServerError as e:
-            raise_server_error(e, ddoc_path)
+        response = self.json_from_resource(ddoc_path, gen=generation)
+        if response == {}:
+            raise InvalidGeneration
+        return response['transaction_id']
 
     def get_replica_gen_and_trans_id(self, other_replica_uid):
         """
@@ -472,11 +450,13 @@ class CouchDatabase(object):
             # TODO: move into resource logic!
             first_entry = self.get_doc(doc_id, check_for_conflicts=True)
             conflicts.append(first_entry)
-        resource = self._database.resource(doc_id, 'u1db_conflicts')
+
         try:
-            response = resource.get_json(**params)
+            response = self.json_from_resource([doc_id, 'u1db_conflicts'],
+                                               check_missing_ddoc=False,
+                                               **params)
             return conflicts + self._build_conflicts(
-                doc_id, json.loads(response[2].read()))
+                doc_id, json.loads(response.read()))
         except ResourceNotFound:
             return []
 
@@ -513,31 +493,13 @@ class CouchDatabase(object):
 
         :return: The complete transaction log.
         :rtype: [(str, str)]
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
         """
         # query a couch view
         ddoc_path = ['_design', 'transactions', '_view', 'log']
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json()
-            return map(
-                lambda row: (row['id'], row['value']),
-                response[2]['rows'])
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
+        response = self.json_from_resource(ddoc_path)
+        return map(
+            lambda row: (row['id'], row['value']),
+            response['rows'])
 
     def whats_changed(self, old_generation=0):
         """
@@ -555,53 +517,33 @@ class CouchDatabase(object):
                  to the last intervening change and sorted by generation (old
                  changes first)
         :rtype: (int, str, [(str, int, str)])
-
-        :raise MissingDesignDocError: Raised when tried to access a missing
-                                      design document.
-        :raise MissingDesignDocListFunctionError: Raised when trying to access
-                                                  a missing list function on a
-                                                  design document.
-        :raise MissingDesignDocNamedViewError: Raised when trying to access a
-                                               missing named view on a design
-                                               document.
-        :raise MissingDesignDocDeletedError: Raised when trying to access a
-                                             deleted design document.
-        :raise MissingDesignDocUnknownError: Raised when failed to access a
-                                             design document for an yet
-                                             unknown reason.
         """
         # query a couch list function
         ddoc_path = [
             '_design', 'transactions', '_list', 'whats_changed', 'log'
         ]
-        res = self._database.resource(*ddoc_path)
-        try:
-            response = res.get_json(old_gen=old_generation)
-            results = map(
-                lambda row:
-                    (row['generation'], row['doc_id'], row['transaction_id']),
-                response[2]['transactions'])
-            results.reverse()
-            cur_gen = old_generation
-            seen = set()
-            changes = []
-            newest_trans_id = ''
-            for generation, doc_id, trans_id in results:
-                if doc_id not in seen:
-                    changes.append((doc_id, generation, trans_id))
-                    seen.add(doc_id)
-            if changes:
-                cur_gen = changes[0][1]  # max generation
-                newest_trans_id = changes[0][2]
-                changes.reverse()
-            else:
-                cur_gen, newest_trans_id = self.get_generation_info()
+        response = self.json_from_resource(ddoc_path, old_gen=old_generation)
+        results = map(
+            lambda row:
+                (row['generation'], row['doc_id'], row['transaction_id']),
+            response['transactions'])
+        results.reverse()
+        cur_gen = old_generation
+        seen = set()
+        changes = []
+        newest_trans_id = ''
+        for generation, doc_id, trans_id in results:
+            if doc_id not in seen:
+                changes.append((doc_id, generation, trans_id))
+                seen.add(doc_id)
+        if changes:
+            cur_gen = changes[0][1]  # max generation
+            newest_trans_id = changes[0][2]
+            changes.reverse()
+        else:
+            cur_gen, newest_trans_id = self.get_generation_info()
 
-            return cur_gen, newest_trans_id, changes
-        except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
-        except ServerError as e:
-            raise_server_error(e, ddoc_path)
+        return cur_gen, newest_trans_id, changes
 
     def get_generation_info(self):
         """
@@ -609,6 +551,25 @@ class CouchDatabase(object):
 
         :return: A tuple containing the current generation and transaction id.
         :rtype: (int, str)
+        """
+        # query a couch list function
+        ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
+        info = self.json_from_resource(ddoc_path)
+        return (info['generation'], info['transaction_id'])
+
+    def json_from_resource(self, ddoc_path, check_missing_ddoc=True,
+                           **kwargs):
+        """
+        Get a resource from it's path and gets a doc's JSON using provided
+        parameters, also checking for missing design docs by default.
+
+        :param ddoc_path: The path to resource.
+        :type ddoc_path: [str]
+        :param check_missing_ddoc: Raises info on what design doc is missing.
+        :type check_missin_ddoc: bool
+
+        :return: The request's data parsed from JSON to a dict.
+        :rtype: dict
 
         :raise MissingDesignDocError: Raised when tried to access a missing
                                       design document.
@@ -624,14 +585,18 @@ class CouchDatabase(object):
                                              design document for an yet
                                              unknown reason.
         """
-        # query a couch list function
-        ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
-        res = self._database.resource(*ddoc_path)
+        if ddoc_path is not None:
+            resource = self._database.resource(*ddoc_path)
+        else:
+            resource = self._database.resource()
         try:
-            response = res.get_json()
-            return (response[2]['generation'], response[2]['transaction_id'])
+            _, _, data = resource.get_json(**kwargs)
+            return data
         except ResourceNotFound as e:
-            raise_missing_design_doc_error(e, ddoc_path)
+            if check_missing_ddoc:
+                raise_missing_design_doc_error(e, ddoc_path)
+            else:
+                raise e
         except ServerError as e:
             raise_server_error(e, ddoc_path)
 
diff --git a/common/src/leap/soledad/common/document.py b/common/src/leap/soledad/common/document.py
index 20510544..9e0c0976 100644
--- a/common/src/leap/soledad/common/document.py
+++ b/common/src/leap/soledad/common/document.py
@@ -132,7 +132,6 @@ class ServerDocument(SoledadDocument):
         :type has_conflicts: bool
         """
         SoledadDocument.__init__(self, doc_id, rev, json, has_conflicts)
-        self.transactions = None
         self._conflicts = None
 
     def get_conflicts(self):
diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py
index 01baf1ce..ccbd6fbd 100644
--- a/server/src/leap/soledad/server/auth.py
+++ b/server/src/leap/soledad/server/auth.py
@@ -189,7 +189,6 @@ class SoledadAuthMiddleware(object):
         @type prefix: str
         """
         self._app = app
-        self._state = app.state
 
     def _error(self, start_response, status, description, message=None):
         """
@@ -350,6 +349,10 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware):
 
     TOKEN_AUTH_ERROR_STRING = "Incorrect address or token."
 
+    def __init__(self, app):
+        self._state = app.state
+        super(SoledadTokenAuthMiddleware, self).__init__(app)
+
     def _verify_authentication_scheme(self, scheme):
         """
         Verify if authentication scheme is valid.
-- 
cgit v1.2.3


From 5e47ff186db38b1f610444f19ec57c838efbbf03 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Thu, 29 Oct 2015 16:56:24 -0300
Subject: [feat] add caching to a place that was missing

When retrieving other replica info the cache wasnt being used, but when
saving it was. This commit applies caching on get as well.
---
 common/src/leap/soledad/common/backend.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
index 2f0ef4f7..deed5ac2 100644
--- a/common/src/leap/soledad/common/backend.py
+++ b/common/src/leap/soledad/common/backend.py
@@ -381,6 +381,8 @@ class SoledadBackend(CommonBackend):
                  synchronized with the replica, this is (0, '').
         :rtype: (int, str)
         """
+        if other_replica_uid in self.cache:
+            return self.cache[other_replica_uid]
         return self._database.get_replica_gen_and_trans_id(other_replica_uid)
 
     def _set_replica_gen_and_trans_id(self, other_replica_uid,
-- 
cgit v1.2.3


From 397343f890dbce9b3d6e9377b54c1f804d39479c Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 3 Nov 2015 12:50:06 -0300
Subject: [style] pep8

---
 common/src/leap/soledad/common/couch/state.py      | 8 ++++----
 common/src/leap/soledad/common/tests/test_couch.py | 3 ++-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py
index 40c0d55b..4f07c105 100644
--- a/common/src/leap/soledad/common/couch/state.py
+++ b/common/src/leap/soledad/common/couch/state.py
@@ -133,10 +133,10 @@ class CouchServerState(ServerState):
         """
         with couch_server(self.couch_url) as server:
             # the tokens db rotates every 30 days, and the current db name is
-            # "tokens_NNN", where NNN is the number of seconds since epoch divided
-            # by the rotate period in seconds. When rotating, old and new tokens
-            # db coexist during a certain window of time and valid tokens are
-            # replicated from the old db to the new one. See:
+            # "tokens_NNN", where NNN is the number of seconds since epoch
+            # divide dby the rotate period in seconds. When rotating, old and
+            # new tokens db coexist during a certain window of time and valid
+            # tokens are replicated from the old db to the new one. See:
             # https://leap.se/code/issues/6785
             dbname = self._tokens_dbname()
             db = server[dbname]
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index a311e0ef..be297b84 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -1437,7 +1437,8 @@ class SoledadBackendExceptionsTests(CouchDBTestCase):
 class DatabaseNameValidationTest(unittest.TestCase):
 
     def test_database_name_validation(self):
-        self.assertFalse(couch.state.is_db_name_valid("user-deadbeef | cat /secret"))
+        inject = couch.state.is_db_name_valid("user-deadbeef | cat /secret")
+        self.assertFalse(inject)
         self.assertTrue(couch.state.is_db_name_valid("user-cafe1337"))
 
 
-- 
cgit v1.2.3


From 0cf6f3efd3cc7dd3361277a8f05577e823e361e5 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 3 Nov 2015 16:53:13 -0300
Subject: [refactor] remove exception based logic

Creating a database was using a unnecessary complex try/except logic.
Simplifying it should make the purpose more clear.
---
 common/src/leap/soledad/common/couch/__init__.py   | 39 ++++++++++++++++------
 common/src/leap/soledad/common/tests/test_couch.py | 19 -----------
 common/src/leap/soledad/common/tests/test_sync.py  |  2 +-
 3 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
index 274f59e2..bd8b08b7 100644
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # __init__.py
-# Copyright (C) 2013 LEAP
+# Copyright (C) 2015 LEAP
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -126,6 +126,8 @@ class CouchDatabase(object):
 
         :return: the database instance
         :rtype: SoledadBackend
+
+        :raise DatabaseDoesNotExist: Raised if database does not exist.
         """
         # get database from url
         m = re.match('(^https?://[^/]+)/(.+)$', url)
@@ -134,12 +136,11 @@ class CouchDatabase(object):
         url = m.group(1)
         dbname = m.group(2)
         with couch_server(url) as server:
-            try:
-                server[dbname]
-            except ResourceNotFound:
-                if not create:
+            if dbname not in server:
+                if create:
+                    server.create(dbname)
+                else:
                     raise DatabaseDoesNotExist()
-                server.create(dbname)
         db = cls(url,
                  dbname, ensure_ddocs=ensure_ddocs,
                  database_security=database_security)
@@ -151,14 +152,32 @@ class CouchDatabase(object):
         self._session = Session(timeout=COUCH_TIMEOUT)
         self._url = url
         self._dbname = dbname
-        self._database = Database(
-            urljoin(url, dbname),
-            self._session)
-        self._database.info()
+        self._database = self.get_couch_database(url, dbname)
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
             self.ensure_security_ddoc(database_security)
 
+    def get_couch_database(self, url, dbname):
+        """
+        Generate a couchdb.Database instance given a url and dbname.
+
+        :param url: CouchDB's server url with credentials
+        :type url: str
+        :param dbname: Database name
+        :type dbname: str
+
+        :return: couch library database instance
+        :rtype: couchdb.Database
+
+        :raise DatabaseDoesNotExist: Raised if database does not exist.
+        """
+        try:
+            return Database(
+                urljoin(url, dbname),
+                self._session)
+        except ResourceNotFound:
+            raise DatabaseDoesNotExist()
+
     def ensure_ddocs_on_db(self):
         """
         Ensure that the design documents used by the backend exist on the
diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py
index be297b84..7ba50e11 100644
--- a/common/src/leap/soledad/common/tests/test_couch.py
+++ b/common/src/leap/soledad/common/tests/test_couch.py
@@ -935,7 +935,6 @@ class SoledadBackendSyncTests(
         doc1 = self.db1.create_doc_from_json(simple_doc)
         doc_id = doc1.doc_id
         doc1_rev = doc1.rev
-        # self.db1.create_index('test-idx', 'key')
         new_doc = '{"key": "altval"}'
         doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id)
         doc2_rev = doc2.rev
@@ -951,12 +950,6 @@ class SoledadBackendSyncTests(
         self.assertTransactionLog([doc_id, doc_id], self.db1)
         self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True)
         self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False)
-        # soledad doesnt support index due to encryption
-        # from_idx = self.db1.get_from_index('test-idx', 'altval')[0]
-        # self.assertEqual(doc2.doc_id, from_idx.doc_id)
-        # self.assertEqual(doc2.rev, from_idx.rev)
-        # self.assertTrue(from_idx.has_conflicts)
-        # self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
 
     def test_sync_sees_remote_delete_conflicted(self):
         self.db1 = self.create_database('test1', 'source')
@@ -989,7 +982,6 @@ class SoledadBackendSyncTests(
         doc = self.db1.create_doc_from_json(simple_doc)
         doc_id = doc.doc_id
         doc1_rev = doc.rev
-        # self.db1.create_index('test-idx', 'key')
         self.sync(self.db1, self.db2)
         content1 = '{"key": "localval"}'
         content2 = '{"key": "altval"}'
@@ -1008,22 +1000,13 @@ class SoledadBackendSyncTests(
         self.sync(self.db1, self.db2, trace_hook=after_whatschanged)
         self.assertEqual([True], triggered)
         self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True)
-        # soledad doesnt support indexing due to encryption
-        # from_idx = self.db1.get_from_index('test-idx', 'altval')[0]
-        # self.assertEqual(doc.doc_id, from_idx.doc_id)
-        # self.assertEqual(doc.rev, from_idx.rev)
-        # self.assertTrue(from_idx.has_conflicts)
-        # self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
-        # self.assertEqual([], self.db1.get_from_index('test-idx', 'localval'))
 
     def test_sync_propagates_deletes(self):
         self.db1 = self.create_database('test1', 'source')
         self.db2 = self.create_database('test2', 'both')
         doc1 = self.db1.create_doc_from_json(simple_doc)
         doc_id = doc1.doc_id
-        # self.db1.create_index('test-idx', 'key')
         self.sync(self.db1, self.db2)
-        # self.db2.create_index('test-idx', 'key')
         self.db3 = self.create_database('test3', 'target')
         self.sync(self.db1, self.db3)
         self.db1.delete_doc(doc1)
@@ -1039,8 +1022,6 @@ class SoledadBackendSyncTests(
             self.db1, doc_id, deleted_rev, None, False)
         self.assertGetDocIncludeDeleted(
             self.db2, doc_id, deleted_rev, None, False)
-        # self.assertEqual([], self.db1.get_from_index('test-idx', 'value'))
-        # self.assertEqual([], self.db2.get_from_index('test-idx', 'value'))
         self.sync(self.db2, self.db3)
         self.assertLastExchangeLog(
             self.db3,
diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py
index 88443179..1041367b 100644
--- a/common/src/leap/soledad/common/tests/test_sync.py
+++ b/common/src/leap/soledad/common/tests/test_sync.py
@@ -101,7 +101,7 @@ class InterruptableSyncTestCase(
             user='user-uuid', server_url=self.getURL())
 
         # ensure remote db exists before syncing
-        db = couch.SoledadBackend.open_database(
+        db = couch.CouchDatabase.open_database(
             urljoin(self.couch_url, 'user-user-uuid'),
             create=True,
             ensure_ddocs=True)
-- 
cgit v1.2.3


From 46678a7aecb98dbdc20846477c7f5110f3c63861 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 17 Nov 2015 12:54:25 -0400
Subject: [feat] add script to measure batch of uploads

- Releases: 0.8.0
---
 scripts/profiling/sync/benchmark-uploads.py | 60 +++++++++++++++++++++++++++++
 scripts/profiling/sync/profile-sync.py      | 11 ++++--
 scripts/profiling/sync/sample               |  7 ++++
 3 files changed, 75 insertions(+), 3 deletions(-)
 create mode 100755 scripts/profiling/sync/benchmark-uploads.py
 create mode 100644 scripts/profiling/sync/sample

diff --git a/scripts/profiling/sync/benchmark-uploads.py b/scripts/profiling/sync/benchmark-uploads.py
new file mode 100755
index 00000000..d371bb1a
--- /dev/null
+++ b/scripts/profiling/sync/benchmark-uploads.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# benchmark_uploads.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Run benchmarks for sync (uploads).
+"""
+import getpass
+import sys
+import datetime
+import subprocess
+
+NUM_ITER = 10
+NUM_UPLOADS = 10
+
+RESULTS = []
+
+
+if __name__ == "__main__":
+    user = sys.argv[1]
+    passwd = sys.argv[2]
+    opts = ("--no-stats --send-num %s --payload-file sample --repeat-payload -p "
+            "%s -b /tmp/profile-soledad-upload ") % (NUM_UPLOADS, passwd)
+
+    for i in xrange(NUM_ITER):
+        print "[+] ITERATION NUMBER: ", i
+        start = datetime.datetime.now()
+        cmd = "./profile-sync.py " + opts + user
+        print "CALLING", cmd
+        result = subprocess.check_call(cmd.split())
+        print "EXIT CODE:", result
+        end = datetime.datetime.now()
+        delta = (end - start)
+        RESULTS.append(delta)
+        print "[+] SYNC TOOK %s seconds" % delta.seconds
+
+    import numpy
+    res = [x.seconds for x in RESULTS]
+    print "-------------------------"
+    print "SYNC UPLOAD REPORT"
+    print "USER:", user
+    print "UPLOADS: %s 1K DOCS" % NUM_UPLOADS
+    print "ITERATIONS: %s" % NUM_ITER
+    print 
+    print "mean :", numpy.mean(res)
+    print "stdev:", numpy.std(res)
+    print "-------------------------"
diff --git a/scripts/profiling/sync/profile-sync.py b/scripts/profiling/sync/profile-sync.py
index 9ef2ea92..8c18bde8 100755
--- a/scripts/profiling/sync/profile-sync.py
+++ b/scripts/profiling/sync/profile-sync.py
@@ -57,15 +57,17 @@ def create_docs(soledad, args):
         bail('--payload-file does not exist!')
         return
 
-    numdocs = args.send_num
-    docsize = args.send_size
+    numdocs = int(args.send_num)
+    docsize = int(args.send_size)
 
     # XXX this will FAIL if the payload source is smaller to size * num
     # XXX could use a cycle iterator
     with open(sample_path, "r+b") as sample_f:
         fmap = mmap.mmap(sample_f.fileno(), 0, prot=mmap.PROT_READ)
+        payload = fmap.read(docsize * 1024)
         for index in xrange(numdocs):
-            payload = fmap.read(docsize * 1024)
+            if not args.repeat_payload:
+                payload = fmap.read(docsize * 1024)
             s.create_doc({payload: payload})
 
 # main program
@@ -94,6 +96,9 @@ if __name__ == '__main__':
     parser.add_argument(
         '--send-num', dest='send_num', default=10,
         help='number of docs to send (default: 10)')
+    parser.add_argument(
+        '--repeat-payload', dest='repeat_payload', action='store_true',
+        default=False)
     parser.add_argument(
         '--payload-file', dest="payload_f", default=None,
         help='path to a sample file to use for the payloads')
diff --git a/scripts/profiling/sync/sample b/scripts/profiling/sync/sample
new file mode 100644
index 00000000..7b8fc8fa
--- /dev/null
+++ b/scripts/profiling/sync/sample
@@ -0,0 +1,7 @@
+    CAOS  NUNCA  MURIO.   Bloque primordial  sin esculpir,  único  excelentísimo  monstruo, inerte  y espontáneo, más ultravioleta que  ninguna ideología  (como las sombras antes de Babilonia), la  homogénea unidad original  del ser todavía irradia serena como los negros pendones de los Asesinos, perpetua y azarosamente ebria.
+
+    Caos precede a todo principio de orden y entropía, no es ni Dios ni  gusano, sus  deseos insensatos abarcan y definen toda posible coreografía,  todo  éter  y  flogisto sin  sentido:  sus  máscaras son cristalizaciones de su propia  falta de rostro,   como   las   nubes.
+
+  Todo en  la naturaleza es perfectamente  real incluyendo la conciencia, no hay absolutamente nada de lo que preocuparse.  No sólo se han roto  las cadenas de la  Ley, es que nunca  existieron; los demonios nunca guardaron las estrellas, el Imperio jamás se fundó, a Eros nunca le creció la barba.
+
+No,  escucha, lo  que ocurrió  fue esto:  te mintieron,  te vendieron ideas  sobre el bien  y el mal
-- 
cgit v1.2.3


From bb228d13aa14cabe7448f9c7e5b55744de36843a Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 19 Nov 2015 18:22:14 -0400
Subject: [bug] fix indirect dependency on couchdb for client

by subclassing the MissingDesignDocError, we don't have to import the
soledad.common.couch submodule into the soledad.client.sync

- Resolves: #7626
---
 client/src/leap/soledad/client/sync.py         |  4 ++--
 common/changes/next-changelog.rst              | 29 ++++++++++++++++++++++++++
 common/src/leap/soledad/common/couch/errors.py |  4 ++--
 common/src/leap/soledad/common/errors.py       |  8 +++++++
 4 files changed, 41 insertions(+), 4 deletions(-)
 create mode 100644 common/changes/next-changelog.rst

diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py
index 626ad2e5..1879031f 100644
--- a/client/src/leap/soledad/client/sync.py
+++ b/client/src/leap/soledad/client/sync.py
@@ -22,7 +22,7 @@ import logging
 from twisted.internet import defer
 
 from u1db import errors
-from leap.soledad.common.couch.errors import MissingDesignDocError
+from leap.soledad.common.errors import BackendNotReadyError
 from u1db.sync import Synchronizer
 
 
@@ -74,7 +74,7 @@ class SoledadSynchronizer(Synchronizer):
             (self.target_replica_uid, target_gen, target_trans_id,
              target_my_gen, target_my_trans_id) = yield \
                 sync_target.get_sync_info(self.source._replica_uid)
-        except (errors.DatabaseDoesNotExist, MissingDesignDocError) as e:
+        except (errors.DatabaseDoesNotExist, BackendNotReadyError) as e:
             logger.debug("Database isn't ready on server. Will be created.")
             logger.debug("Reason: %s", e.__class__)
             self.target_replica_uid = None
diff --git a/common/changes/next-changelog.rst b/common/changes/next-changelog.rst
new file mode 100644
index 00000000..e05f80ce
--- /dev/null
+++ b/common/changes/next-changelog.rst
@@ -0,0 +1,29 @@
+0.8.0 - xxx
++++++++++++++++++++++++++++++++
+
+Please add lines to this file, they will be moved to the CHANGELOG.rst during
+the next release.
+
+There are two template lines for each category, use them as reference.
+
+I've added a new category `Misc` so we can track doc/style/packaging stuff.
+
+Features
+~~~~~~~~
+- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
+- New feature without related issue number.
+
+Bugfixes
+~~~~~~~~
+- `#7626 <https://leap.se/code/issues/7626>`_: Subclass a leaky leap.common.couch exception to avoid depending on couch.
+- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235.
+- Bugfix without related issue number.
+
+Misc
+~~~~
+- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
+- Some change without issue number.
+
+Known Issues
+~~~~~~~~~~~~
+- `#1236 <https://leap.se/code/issues/1236>`_: Description of the known issue corresponding with issue #1236.
diff --git a/common/src/leap/soledad/common/couch/errors.py b/common/src/leap/soledad/common/couch/errors.py
index e894d58f..9b287c76 100644
--- a/common/src/leap/soledad/common/couch/errors.py
+++ b/common/src/leap/soledad/common/couch/errors.py
@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
-from leap.soledad.common.errors import SoledadError
+from leap.soledad.common.errors import SoledadError, BackendNotReadyError
 from leap.soledad.common.errors import register_exception
 
 """
@@ -23,7 +23,7 @@ Specific errors that can be raised by CouchDatabase.
 
 
 @register_exception
-class MissingDesignDocError(SoledadError):
+class MissingDesignDocError(BackendNotReadyError):
 
     """
     Raised when trying to access a missing couch design document.
diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py
index 2f6fd1d8..7a8a8929 100644
--- a/common/src/leap/soledad/common/errors.py
+++ b/common/src/leap/soledad/common/errors.py
@@ -143,3 +143,11 @@ class InvalidURLError(Exception):
     """
     Exception raised when Soledad encounters a malformed URL.
     """
+
+
+@register_exception
+class BackendNotReadyError(SoledadError):
+    """
+    Generic exception raised when the backend is not ready to dispatch a client
+    request.
+    """
-- 
cgit v1.2.3


From 34731d6208de517be3fa387e57fb90efb265b554 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 20 Nov 2015 15:20:33 -0400
Subject: [bug] do not register exception, breaks

since the exception doesn't have the code and description, it breaks.
we don't need those since the couch child exception describes them.
---
 common/src/leap/soledad/common/errors.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py
index 7a8a8929..f783404c 100644
--- a/common/src/leap/soledad/common/errors.py
+++ b/common/src/leap/soledad/common/errors.py
@@ -145,7 +145,6 @@ class InvalidURLError(Exception):
     """
 
 
-@register_exception
 class BackendNotReadyError(SoledadError):
     """
     Generic exception raised when the backend is not ready to dispatch a client
-- 
cgit v1.2.3


From 086f9fad3ebb1bdcc0ec3fa6161f801068a0c641 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 17 Nov 2015 16:58:15 -0300
Subject: [bug] put a monkeypatch back for bigcouch

Current code was tested on couch 1.6 and a monkeypatch got removed
during refactor. This commit re-adds it, but in a separate module that
is intended to hold temporary code for compatibility that can be removed
on version upgrades.
---
 common/src/leap/soledad/common/couch/__init__.py |   8 +-
 common/src/leap/soledad/common/couch/support.py  | 115 +++++++++++++++++++++++
 2 files changed, 119 insertions(+), 4 deletions(-)
 create mode 100644 common/src/leap/soledad/common/couch/support.py

diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
index bd8b08b7..fb3d57af 100644
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -33,7 +33,6 @@ from multiprocessing.pool import ThreadPool
 
 
 from couchdb.client import Server, Database
-from couchdb.multipart import MultipartWriter
 from couchdb.http import (
     ResourceConflict,
     ResourceNotFound,
@@ -53,6 +52,7 @@ from u1db.remote import http_app
 from leap.soledad.common import ddocs
 from .errors import raise_server_error
 from .errors import raise_missing_design_doc_error
+from .support import MultipartWriter
 from leap.soledad.common.errors import InvalidURLError
 from leap.soledad.common.document import ServerDocument
 from leap.soledad.common.backend import SoledadBackend
@@ -692,8 +692,7 @@ class CouchDatabase(object):
             couch_doc['_rev'] = old_doc.couch_rev
         # prepare the multipart PUT
         buf = StringIO()
-        headers = {}
-        envelope = MultipartWriter(buf, headers=headers, subtype='related')
+        envelope = MultipartWriter(buf)
         envelope.add('application/json', json.dumps(couch_doc))
         for part in parts:
             envelope.add('application/octet-stream', part)
@@ -702,7 +701,8 @@ class CouchDatabase(object):
         try:
             resource = self._new_resource()
             resource.put_json(
-                doc.doc_id, body=str(buf.getvalue()), headers=headers)
+                doc.doc_id, body=str(buf.getvalue()),
+                headers=envelope.headers)
         except ResourceConflict:
             raise RevisionConflict()
         return transactions[-1][1]
diff --git a/common/src/leap/soledad/common/couch/support.py b/common/src/leap/soledad/common/couch/support.py
new file mode 100644
index 00000000..bfc4fef6
--- /dev/null
+++ b/common/src/leap/soledad/common/couch/support.py
@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+# support.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import sys
+
+
+"""
+Monkey patches and temporary code that may be removed with version changes.
+"""
+
+
+# for bigcouch
+# TODO: Remove if bigcouch support is dropped
+class MultipartWriter(object):
+
+    """
+    A multipart writer adapted from python-couchdb's one so we can PUT
+    documents using couch's multipart PUT.
+
+    This stripped down version does not allow for nested structures, and
+    contains only the essential things we need to PUT SoledadDocuments to the
+    couch backend. Also, please note that this is a patch. The couchdb lib has
+    another implementation that works fine with CouchDB 1.6, but removing this
+    now will break compatibility with bigcouch.
+    """
+
+    CRLF = '\r\n'
+
+    def __init__(self, fileobj, headers=None, boundary=None):
+        """
+        Initialize the multipart writer.
+        """
+        self.fileobj = fileobj
+        if boundary is None:
+            boundary = self._make_boundary()
+        self._boundary = boundary
+        self._build_headers('related', headers)
+
+    def add(self, mimetype, content, headers={}):
+        """
+        Add a part to the multipart stream.
+        """
+        self.fileobj.write('--')
+        self.fileobj.write(self._boundary)
+        self.fileobj.write(self.CRLF)
+        headers['Content-Type'] = mimetype
+        self._write_headers(headers)
+        if content:
+            # XXX: throw an exception if a boundary appears in the content??
+            self.fileobj.write(content)
+            self.fileobj.write(self.CRLF)
+
+    def close(self):
+        """
+        Close the multipart stream.
+        """
+        self.fileobj.write('--')
+        self.fileobj.write(self._boundary)
+        # be careful not to have anything after '--', otherwise old couch
+        # versions (including bigcouch) will fail.
+        self.fileobj.write('--')
+
+    def _make_boundary(self):
+        """
+        Create a boundary to discern multi parts.
+        """
+        try:
+            from uuid import uuid4
+            return '==' + uuid4().hex + '=='
+        except ImportError:
+            from random import randrange
+            token = randrange(sys.maxint)
+            format = '%%0%dd' % len(repr(sys.maxint - 1))
+            return '===============' + (format % token) + '=='
+
+    def _write_headers(self, headers):
+        """
+        Write a part header in the buffer stream.
+        """
+        if headers:
+            for name in sorted(headers.keys()):
+                value = headers[name]
+                self.fileobj.write(name)
+                self.fileobj.write(': ')
+                self.fileobj.write(value)
+                self.fileobj.write(self.CRLF)
+        self.fileobj.write(self.CRLF)
+
+    def _build_headers(self, subtype, headers):
+        """
+        Build the main headers of the multipart stream.
+
+        This is here so we can send headers separete from content using
+        python-couchdb API.
+        """
+        self.headers = {}
+        self.headers['Content-Type'] = 'multipart/%s; boundary="%s"' % \
+                                       (subtype, self._boundary)
+        if headers:
+            for name in sorted(headers.keys()):
+                value = headers[name]
+                self.headers[name] = value
-- 
cgit v1.2.3


From 3537351ad94dd17d0c0a57ec77489b4a540b3d23 Mon Sep 17 00:00:00 2001
From: Bruno Wagner <bwagner@riseup.net>
Date: Fri, 20 Nov 2015 11:02:14 -0200
Subject: [bug] BackendNotReadyError breaks without status

The new BackendNotReadyError didn't have a status or a wire
description, because of that, when you tried to use the
leap.soledad.server package it would break trying to import
this exception (because the annotation tries to use this
variable). This was preventing soledad server from starting
at all, after this change it works again
---
 common/src/leap/soledad/common/errors.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py
index f783404c..0b6bb4e6 100644
--- a/common/src/leap/soledad/common/errors.py
+++ b/common/src/leap/soledad/common/errors.py
@@ -150,3 +150,5 @@ class BackendNotReadyError(SoledadError):
     Generic exception raised when the backend is not ready to dispatch a client
     request.
     """
+    wire_description = "backend not ready"
+    status = 500
-- 
cgit v1.2.3


From b44a2a9b3894e438bbb2b18da6e22f86602655af Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 24 Nov 2015 22:45:34 -0300
Subject: [bug] generation_info cant be cached per replica

This info can be changed by another syncing replica and would not
reflect real database generation. That would be ok inside of the same
sync, but can cause trouble on concurrent syncs.
The other calls are ok, since they hold info that doesnt change during
concurrent syncs or are only read/write by the replica syncing. A global
cache could fit better this removed case, but for now let's stay on the
safe side.
---
 common/src/leap/soledad/common/backend.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
index deed5ac2..a1493adc 100644
--- a/common/src/leap/soledad/common/backend.py
+++ b/common/src/leap/soledad/common/backend.py
@@ -154,11 +154,7 @@ class SoledadBackend(CommonBackend):
 
         :raise SoledadError: Raised by database on operation failure
         """
-        if self.replica_uid + '_gen' in self.cache:
-            response = self.cache[self.replica_uid + '_gen']
-            return response
         cur_gen, newest_trans_id = self._database.get_generation_info()
-        self.cache[self.replica_uid + '_gen'] = (cur_gen, newest_trans_id)
         return (cur_gen, newest_trans_id)
 
     def _get_trans_id_for_gen(self, generation):
@@ -253,14 +249,8 @@ class SoledadBackend(CommonBackend):
         :param doc: The document to be put.
         :type doc: ServerDocument
         """
-        last_transaction =\
-            self._database.save_document(old_doc, doc,
-                                         self._allocate_transaction_id())
-        if self.replica_uid + '_gen' in self.cache:
-            gen, trans = self.cache[self.replica_uid + '_gen']
-            gen += 1
-            trans = last_transaction
-            self.cache[self.replica_uid + '_gen'] = (gen, trans)
+        self._database.save_document(old_doc, doc,
+                                     self._allocate_transaction_id())
 
     def put_doc(self, doc):
         """
-- 
cgit v1.2.3


From 332ce9e68a34e2ddc31ef371c991565e5e38812b Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 1 Dec 2015 13:07:14 -0300
Subject: [tests] encdec pool is being reused, adding a test

This new test case will run the single insert test 5 times to ensure
that using the same pool again is fine. This is needed due failures
to shutdown the pool or inconsistency between syncs.
---
 common/src/leap/soledad/common/tests/test_encdecpool.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/common/src/leap/soledad/common/tests/test_encdecpool.py b/common/src/leap/soledad/common/tests/test_encdecpool.py
index 793bfa1a..27247c94 100644
--- a/common/src/leap/soledad/common/tests/test_encdecpool.py
+++ b/common/src/leap/soledad/common/tests/test_encdecpool.py
@@ -171,12 +171,22 @@ class TestSyncDecrypterPool(BaseSoledadTest):
             DOC_ID, DOC_REV, encrypted_content, 1, "trans_id", 1)
 
         def _assert_doc_was_decrypted_and_inserted(_):
+            self.assertEqual(1, len(self._inserted_docs))
             self.assertEqual(self._inserted_docs, [(doc, 1, u"trans_id")])
 
         self._pool.deferred.addCallback(
             _assert_doc_was_decrypted_and_inserted)
         return self._pool.deferred
 
+    @inlineCallbacks
+    def test_pool_reuse(self):
+        """
+        The pool is reused between syncs, this test verifies that
+        reusing is fine.
+        """
+        for _ in xrange(5):
+            yield self.test_insert_encrypted_received_doc()
+
     def test_insert_encrypted_received_doc_many(self):
         """
         Test that many encrypted documents added to the pool are decrypted and
-- 
cgit v1.2.3


From 55f45b770a57d1c5f54a66a490aeeea7edae0184 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 1 Dec 2015 15:58:57 -0300
Subject: [bug] fire callback after reseting instance vars

If we reset the vars after firing the finish callback, other thread can pick
up a dirty state on due concurrency.
---
 client/src/leap/soledad/client/encdecpool.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py
index 6d3c11b9..9333578b 100644
--- a/client/src/leap/soledad/client/encdecpool.py
+++ b/client/src/leap/soledad/client/encdecpool.py
@@ -807,6 +807,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
             self._finish()
 
     def _finish(self):
-        self._deferred.callback(None)
         self._processed_docs = 0
         self._last_inserted_idx = 0
+        self._deferred.callback(None)
-- 
cgit v1.2.3


From 202bdc553cc576bfbce1ba8a4c34569b1751c04d Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 1 Dec 2015 16:02:47 -0300
Subject: [test] shuffle and decrypts 5 docs 3 times

On real usage the docs will arrive shuffled and pool will be reused
after many decrypts. This test asserts that everything ended up clear
between execution and no inconsistency is left over for the next run.
---
 .../leap/soledad/common/tests/test_encdecpool.py   | 34 +++++++++++++---------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/common/src/leap/soledad/common/tests/test_encdecpool.py b/common/src/leap/soledad/common/tests/test_encdecpool.py
index 27247c94..694eb7ad 100644
--- a/common/src/leap/soledad/common/tests/test_encdecpool.py
+++ b/common/src/leap/soledad/common/tests/test_encdecpool.py
@@ -18,6 +18,7 @@
 Tests for encryption and decryption pool.
 """
 import json
+from random import shuffle
 
 from twisted.internet.defer import inlineCallbacks
 
@@ -178,23 +179,14 @@ class TestSyncDecrypterPool(BaseSoledadTest):
             _assert_doc_was_decrypted_and_inserted)
         return self._pool.deferred
 
-    @inlineCallbacks
-    def test_pool_reuse(self):
-        """
-        The pool is reused between syncs, this test verifies that
-        reusing is fine.
-        """
-        for _ in xrange(5):
-            yield self.test_insert_encrypted_received_doc()
-
-    def test_insert_encrypted_received_doc_many(self):
+    def test_insert_encrypted_received_doc_many(self, many=100):
         """
         Test that many encrypted documents added to the pool are decrypted and
         inserted using the callback.
         """
         crypto = self._soledad._crypto
-        many = 100
         self._pool.start(many)
+        docs = []
 
         # insert many encrypted docs in the pool
         for i in xrange(many):
@@ -208,9 +200,12 @@ class TestSyncDecrypterPool(BaseSoledadTest):
                 doc_id=doc_id, rev=rev, json=json.dumps(content))
 
             encrypted_content = json.loads(crypto.encrypt_doc(doc))
+            docs.append((doc_id, rev, encrypted_content, gen,
+                         trans_id, idx))
+        shuffle(docs)
 
-            self._pool.insert_encrypted_received_doc(
-                doc_id, rev, encrypted_content, gen, trans_id, idx)
+        for doc in docs:
+            self._pool.insert_encrypted_received_doc(*doc)
 
         def _assert_docs_were_decrypted_and_inserted(_):
             self.assertEqual(many, len(self._inserted_docs))
@@ -233,3 +228,16 @@ class TestSyncDecrypterPool(BaseSoledadTest):
         self._pool.deferred.addCallback(
             _assert_docs_were_decrypted_and_inserted)
         return self._pool.deferred
+
+    @inlineCallbacks
+    def test_pool_reuse(self):
+        """
+        The pool is reused between syncs, this test verifies that
+        reusing is fine.
+        """
+        for i in xrange(3):
+            yield self.test_insert_encrypted_received_doc_many(5)
+            self._inserted_docs = []
+            decrypted_docs = yield self._pool._get_docs(encrypted=False)
+            # check that decrypted docs staging is clean
+            self.assertEquals([], decrypted_docs)
-- 
cgit v1.2.3


From 841e6712ff9ff1ce2b8a5fe92012d89c2aec7077 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 1 Dec 2015 16:06:26 -0300
Subject: [bug] concurrency bug while querying and inserting

This line was missing an yield and without it we end up inserting a
document that is being retrieved and bad things happen.
This is the core fix from yesterday debugging session. During sequential
syncs the pool was inserting and querying at the same time and sometimes
repeating or failing to delete documents.
---
 client/src/leap/soledad/client/encdecpool.py | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py
index 9333578b..a01d3b71 100644
--- a/client/src/leap/soledad/client/encdecpool.py
+++ b/client/src/leap/soledad/client/encdecpool.py
@@ -70,11 +70,13 @@ class SyncEncryptDecryptPool(object):
         self._started = False
 
     def start(self):
+        if self.running:
+            return
         self._create_pool()
         self._started = True
 
     def stop(self):
-        if not self._started:
+        if not self.running:
             return
         self._started = False
         self._destroy_pool()
@@ -650,14 +652,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         last_idx = self._last_inserted_idx
         for doc_id, rev, content, gen, trans_id, encrypted, idx in \
                 decrypted_docs:
-            # XXX for some reason, a document might not have been deleted from
-            #     the database. This is a bug. In this point, already
-            #     processed documents should have been removed from the sync
-            #     database and we should not have to skip them here. We need
-            #     to find out why this is happening, fix, and remove the
-            #     skipping below.
-            if (idx < last_idx + 1):
-                continue
             if (idx != last_idx + 1):
                 break
             insertable.append((doc_id, rev, content, gen, trans_id, idx))
@@ -763,6 +757,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         query = "DELETE FROM %s WHERE 1" % (self.TABLE_NAME,)
         return self._runOperation(query)
 
+    @defer.inlineCallbacks
     def _collect_async_decryption_results(self):
         """
         Collect the results of the asynchronous doc decryptions and re-raise
@@ -773,7 +768,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         async_results = self._async_results[:]
         for res in async_results:
             if res.ready():
-                self._decrypt_doc_cb(res.get())  # might raise an exception!
+                yield self._decrypt_doc_cb(res.get())  # might raise an exception!
                 self._async_results.remove(res)
 
     @defer.inlineCallbacks
@@ -796,7 +791,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
 
         if processed < pending:
             yield self._async_decrypt_received_docs()
-            self._collect_async_decryption_results()
+            yield self._collect_async_decryption_results()
             docs = yield self._process_decrypted_docs()
             yield self._delete_processed_docs(docs)
             # recurse
-- 
cgit v1.2.3


From 3f327c4f472f43d281e3bd7be67aaa9ce3f7d822 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 1 Dec 2015 16:38:02 -0300
Subject: [style] fix pep8

---
 client/src/leap/soledad/client/encdecpool.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py
index a01d3b71..0954c1df 100644
--- a/client/src/leap/soledad/client/encdecpool.py
+++ b/client/src/leap/soledad/client/encdecpool.py
@@ -768,7 +768,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         async_results = self._async_results[:]
         for res in async_results:
             if res.ready():
-                yield self._decrypt_doc_cb(res.get())  # might raise an exception!
+                # XXX: might raise an exception!
+                yield self._decrypt_doc_cb(res.get())
                 self._async_results.remove(res)
 
     @defer.inlineCallbacks
-- 
cgit v1.2.3


From 129db70b5237ffbc7b38d9931598629f46ce4763 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 2 Dec 2015 16:16:27 -0400
Subject: [docs] fix run-tests snippet

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 815db808..74a49d12 100644
--- a/README.rst
+++ b/README.rst
@@ -51,7 +51,7 @@ Client and server tests are both included in leap.soledad.common. If you want
 to run tests in development mode you must do the following::
 
   scripts/develop_mode.sh
-  ./run_tests.sh
+  scripts/run_tests.sh
 
 Note that to run CouchDB tests, be sure you have `CouchDB`_ installed on your
 system.
-- 
cgit v1.2.3


From 6fd543b9fd9679f4978aeedee32eeece5593acc3 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 13 Nov 2015 22:10:18 -0300
Subject: [feat] Adds support to batching limited by size

u1db provides batching by default. Current Soledad HTTPS Sync Target was
stuck at 1 doc per request. This commit adds batching capability,
limiting the size to a predefined value.
Default limit size: 500kB
---
 client/src/leap/soledad/client/http_target/send.py | 46 +++++++++++++++-------
 .../src/leap/soledad/client/http_target/support.py | 13 ++++--
 2 files changed, 41 insertions(+), 18 deletions(-)

diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index 80483f0d..c1252c13 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -29,6 +29,8 @@ class HTTPDocSender(object):
     They need to be encrypted and metadata prepared before sending.
     """
 
+    MAX_BATCH_SIZE = 500 * 1000  # 500kB by default
+
     @defer.inlineCallbacks
     def _send_docs(self, docs_by_generation, last_known_generation,
                    last_known_trans_id, sync_id):
@@ -43,25 +45,41 @@ class HTTPDocSender(object):
             sync_id=sync_id,
             ensure=self._ensure_callback is not None)
         total = len(docs_by_generation)
-        for idx, entry in enumerate(docs_by_generation, 1):
-            yield self._prepare_one_doc(entry, body, idx, total)
-            result = yield self._http_request(
-                self._url,
-                method='POST',
-                body=body.pop(1),
-                content_type='application/x-soledad-sync-put')
-            if self._defer_encryption:
-                self._delete_sent(idx, docs_by_generation)
-            _emit_send_status(idx, total)
+        while body.consumed < total:
+            result = yield self._send_batch(total, body, docs_by_generation)
         response_dict = json.loads(result)[0]
         gen_after_send = response_dict['new_generation']
         trans_id_after_send = response_dict['new_transaction_id']
         defer.returnValue([gen_after_send, trans_id_after_send])
 
-    def _delete_sent(self, idx, docs_by_generation):
-        doc = docs_by_generation[idx - 1][0]
-        self._sync_enc_pool.delete_encrypted_doc(
-            doc.doc_id, doc.rev)
+    def _delete_sent(self, docs):
+        for doc, gen, trans_id in docs:
+            self._sync_enc_pool.delete_encrypted_doc(
+                doc.doc_id, doc.rev)
+
+    @defer.inlineCallbacks
+    def _send_batch(self, total, body, docs):
+        sent = []
+        missing = total - body.consumed
+        for i in xrange(1, missing + 1):
+            if body.pending_size > self.MAX_BATCH_SIZE:
+                break
+            idx = body.consumed + i
+            entry = docs[idx - 1]
+            sent.append(entry)
+            yield self._prepare_one_doc(entry, body, idx, total)
+        result = yield self._send_request(body.pop())
+        if self._defer_encryption:
+            self._delete_sent(sent)
+        _emit_send_status(body.consumed, total)
+        defer.returnValue(result)
+
+    def _send_request(self, body):
+        return self._http_request(
+            self._url,
+            method='POST',
+            body=body,
+            content_type='application/x-soledad-sync-put')
 
     @defer.inlineCallbacks
     def _prepare_one_doc(self, entry, body, idx, total):
diff --git a/client/src/leap/soledad/client/http_target/support.py b/client/src/leap/soledad/client/http_target/support.py
index 44cd7089..2625744c 100644
--- a/client/src/leap/soledad/client/http_target/support.py
+++ b/client/src/leap/soledad/client/http_target/support.py
@@ -152,6 +152,8 @@ class RequestBody(object):
         """
         self.headers = header_dict
         self.entries = []
+        self.consumed = 0
+        self.pending_size = 0
 
     def insert_info(self, **entry_dict):
         """
@@ -165,11 +167,11 @@ class RequestBody(object):
         """
         entry = json.dumps(entry_dict)
         self.entries.append(entry)
-        return len(entry)
+        self.pending_size += len(entry)
 
-    def pop(self, number=1):
+    def pop(self):
         """
-        Removes an amount of entries and returns it formatted and ready
+        Removes all entries and returns it formatted and ready
         to be sent.
 
         :param number: number of entries to pop and format
@@ -178,7 +180,10 @@ class RequestBody(object):
         :return: formatted body ready to be sent
         :rtype: str
         """
-        entries = [self.entries.pop(0) for i in xrange(number)]
+        entries = self.entries[:]
+        self.entries = []
+        self.pending_size = 0
+        self.consumed += len(entries)
         return self.entries_to_str(entries)
 
     def __str__(self):
-- 
cgit v1.2.3


From 577abee147c98592753bcdc68e1693d1f4ab5a08 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 13 Nov 2015 23:02:28 -0300
Subject: [feat] prepare server to handle batches

Created two methods on the backend to start and finish a batch. A dict of
callbacks is available to defer actions for the last document, allowing
temporary (changing often) metadata to be recorded only once.
Using those methods we will also be able to put all docs in one go on
the CouchDatabase implementation, but that is another step.
---
 client/changes/feat_send_batch               |  1 +
 common/src/leap/soledad/common/backend.py    | 26 ++++++++++++++++++++++----
 server/changes/feat_handle_send_batch_better |  1 +
 server/src/leap/soledad/server/sync.py       | 14 +++++++++++---
 4 files changed, 35 insertions(+), 7 deletions(-)
 create mode 100644 client/changes/feat_send_batch
 create mode 100644 server/changes/feat_handle_send_batch_better

diff --git a/client/changes/feat_send_batch b/client/changes/feat_send_batch
new file mode 100644
index 00000000..fbfce519
--- /dev/null
+++ b/client/changes/feat_send_batch
@@ -0,0 +1 @@
+o Client will now send documents at a limited size batch due to changes on SyncTarget. The default limit is 500kB.
diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
index a1493adc..b083163e 100644
--- a/common/src/leap/soledad/common/backend.py
+++ b/common/src/leap/soledad/common/backend.py
@@ -53,9 +53,20 @@ class SoledadBackend(CommonBackend):
         self._cache = None
         self._dbname = database._dbname
         self._database = database
+        self.batching = False
         if replica_uid is not None:
             self._set_replica_uid(replica_uid)
 
+    def batch_start(self):
+        self.batching = True
+        self.after_batch_callbacks = {}
+
+    def batch_end(self):
+        self.batching = False
+        for name in self.after_batch_callbacks:
+            self.after_batch_callbacks[name]()
+        self.after_batch_callbacks = None
+
     @property
     def cache(self):
         if self._cache is not None:
@@ -373,7 +384,10 @@ class SoledadBackend(CommonBackend):
         """
         if other_replica_uid in self.cache:
             return self.cache[other_replica_uid]
-        return self._database.get_replica_gen_and_trans_id(other_replica_uid)
+        gen, trans_id = \
+            self._database.get_replica_gen_and_trans_id(other_replica_uid)
+        self.cache[other_replica_uid] = (gen, trans_id)
+        return (gen, trans_id)
 
     def _set_replica_gen_and_trans_id(self, other_replica_uid,
                                       other_generation, other_transaction_id):
@@ -413,9 +427,13 @@ class SoledadBackend(CommonBackend):
                                      generation.
         :type other_transaction_id: str
         """
-        self._set_replica_gen_and_trans_id(other_replica_uid,
-                                           other_generation,
-                                           other_transaction_id)
+        function = self._set_replica_gen_and_trans_id
+        args = [other_replica_uid, other_generation, other_transaction_id]
+        callback = lambda: function(*args)
+        if self.batching:
+            self.after_batch_callbacks['set_source_info'] = callback
+        else:
+            callback()
 
     def _force_doc_sync_conflict(self, doc):
         """
diff --git a/server/changes/feat_handle_send_batch_better b/server/changes/feat_handle_send_batch_better
new file mode 100644
index 00000000..6ee8688a
--- /dev/null
+++ b/server/changes/feat_handle_send_batch_better
@@ -0,0 +1 @@
+o Added two methods to start and finish a batch on backend. They can be used to change database behaviour, allowing batch operations to be optimized.
diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py
index db25c406..96f65912 100644
--- a/server/src/leap/soledad/server/sync.py
+++ b/server/src/leap/soledad/server/sync.py
@@ -112,6 +112,14 @@ class SyncExchange(sync.SyncExchange):
             doc = self._db.get_doc(changed_doc_id, include_deleted=True)
             return_doc_cb(doc, gen, trans_id)
 
+    def batched_insert_from_source(self, entries, sync_id):
+        self._db.batch_start()
+        for entry in entries:
+            doc, gen, trans_id, number_of_docs, doc_idx = entry
+            self.insert_doc_from_source(doc, gen, trans_id, number_of_docs,
+                                        doc_idx, sync_id)
+        self._db.batch_end()
+
     def insert_doc_from_source(
             self, doc, source_gen, trans_id,
             number_of_docs=None, doc_idx=None, sync_id=None):
@@ -198,6 +206,7 @@ class SyncResource(http_app.SyncResource):
         self.sync_exch = self.sync_exchange_class(
             db, self.source_replica_uid, last_known_generation, sync_id)
         self._sync_id = sync_id
+        self._staging = []
 
     @http_app.http_method(content_as_args=True)
     def post_put(
@@ -225,9 +234,7 @@ class SyncResource(http_app.SyncResource):
         :type doc_idx: int
         """
         doc = Document(id, rev, content)
-        self.sync_exch.insert_doc_from_source(
-            doc, gen, trans_id, number_of_docs=number_of_docs,
-            doc_idx=doc_idx, sync_id=self._sync_id)
+        self._staging.append((doc, gen, trans_id, number_of_docs, doc_idx))
 
     @http_app.http_method(received=int, content_as_args=True)
     def post_get(self, received):
@@ -266,6 +273,7 @@ class SyncResource(http_app.SyncResource):
         Return the current generation and transaction_id after inserting one
         incoming document.
         """
+        self.sync_exch.batched_insert_from_source(self._staging, self._sync_id)
         self.responder.content_type = 'application/x-soledad-sync-response'
         self.responder.start_response(200)
         self.responder.start_stream(),
-- 
cgit v1.2.3


From bcbb9ccd1d4a281b6922340c12ec01b09d636380 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sat, 14 Nov 2015 03:38:20 -0300
Subject: [feat] put all docs at once

Using _bulk_docs api from CouchDB we can put all docs at a single
request. Also, prefetching all ids removes the need to HEAD
requests during the batch.
---
 common/src/leap/soledad/common/backend.py        |  6 +++
 common/src/leap/soledad/common/couch/__init__.py | 51 +++++++++++++++++-------
 2 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
index b083163e..9f5950b2 100644
--- a/common/src/leap/soledad/common/backend.py
+++ b/common/src/leap/soledad/common/backend.py
@@ -60,9 +60,15 @@ class SoledadBackend(CommonBackend):
     def batch_start(self):
         self.batching = True
         self.after_batch_callbacks = {}
+        self._database.batch_start()
+        if not self._cache:
+            # batching needs cache
+            self._cache = {}
+        self._get_generation()  # warm up gen info
 
     def batch_end(self):
         self.batching = False
+        self._database.batch_end()
         for name in self.after_batch_callbacks:
             self.after_batch_callbacks[name]()
         self.after_batch_callbacks = None
diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
index fb3d57af..f3437ae1 100644
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -153,10 +153,22 @@ class CouchDatabase(object):
         self._url = url
         self._dbname = dbname
         self._database = self.get_couch_database(url, dbname)
+        self.batching = False
+        self.batch_docs = []
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
             self.ensure_security_ddoc(database_security)
 
+    def batch_start(self):
+        self.batching = True
+        ids = set(row.id for row in self._database.view('_all_docs'))
+        self.batched_ids = ids
+
+    def batch_end(self):
+        self.batching = False
+        self._database.update(self.batch_docs)
+        self.batch_docs = []
+
     def get_couch_database(self, url, dbname):
         """
         Generate a couchdb.Database instance given a url and dbname.
@@ -339,6 +351,8 @@ class CouchDatabase(object):
         """
         # get document with all attachments (u1db content and eventual
         # conflicts)
+        if self.batching and doc_id not in self.batched_ids:
+            return None
         if doc_id not in self._database:
             return None
         result = self.json_from_resource([doc_id], attachments=True)
@@ -691,20 +705,29 @@ class CouchDatabase(object):
         if old_doc is not None and hasattr(old_doc, 'couch_rev'):
             couch_doc['_rev'] = old_doc.couch_rev
         # prepare the multipart PUT
-        buf = StringIO()
-        envelope = MultipartWriter(buf)
-        envelope.add('application/json', json.dumps(couch_doc))
-        for part in parts:
-            envelope.add('application/octet-stream', part)
-        envelope.close()
-        # try to save and fail if there's a revision conflict
-        try:
-            resource = self._new_resource()
-            resource.put_json(
-                doc.doc_id, body=str(buf.getvalue()),
-                headers=envelope.headers)
-        except ResourceConflict:
-            raise RevisionConflict()
+        if not self.batching:
+            buf = StringIO()
+            envelope = MultipartWriter(buf)
+            envelope.add('application/json', json.dumps(couch_doc))
+            for part in parts:
+                envelope.add('application/octet-stream', part)
+            envelope.close()
+            # try to save and fail if there's a revision conflict
+            try:
+                resource = self._new_resource()
+                resource.put_json(
+                    doc.doc_id, body=str(buf.getvalue()),
+                    headers=envelope.headers)
+            except ResourceConflict:
+                raise RevisionConflict()
+        else:
+            for name, attachment in attachments.items():
+                del attachment['follows']
+                del attachment['length']
+                index = 0 if name is 'u1db_content' else 1
+                attachment['data'] = binascii.b2a_base64(parts[index]).strip()
+            couch_doc['attachments'] = attachments
+            self.batch_docs.append(couch_doc)
         return transactions[-1][1]
 
     def _new_resource(self, *path):
-- 
cgit v1.2.3


From fa595231b5ea346dfd9e06c364854469397fca3f Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Sat, 14 Nov 2015 21:53:14 -0300
Subject: [feat] checks staged docs inside batch

This commit adds checking for consistency on batch. When a doc is needed
during a batched sync and it doesnt exists on database, current code
will make a partial batch to avoid processing like it doesnt exist.
---
 common/src/leap/soledad/common/couch/__init__.py | 50 ++++++++++++++++++++----
 1 file changed, 42 insertions(+), 8 deletions(-)

diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
index f3437ae1..a922fd48 100644
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -154,7 +154,7 @@ class CouchDatabase(object):
         self._dbname = dbname
         self._database = self.get_couch_database(url, dbname)
         self.batching = False
-        self.batch_docs = []
+        self.batch_docs = {}
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
             self.ensure_security_ddoc(database_security)
@@ -166,8 +166,7 @@ class CouchDatabase(object):
 
     def batch_end(self):
         self.batching = False
-        self._database.update(self.batch_docs)
-        self.batch_docs = []
+        self.__perform_batch()
 
     def get_couch_database(self, url, dbname):
         """
@@ -197,7 +196,8 @@ class CouchDatabase(object):
         """
         for ddoc_name in ['docs', 'syncs', 'transactions']:
             try:
-                self.json_from_resource(['_design', ddoc_name, '_info'],
+                self.json_from_resource(['_design'] +
+                                        ddoc_name.split('/') + ['_info'],
                                         check_missing_ddoc=False)
             except ResourceNotFound:
                 ddoc = json.loads(
@@ -349,15 +349,49 @@ class CouchDatabase(object):
         :return: The document.
         :rtype: ServerDocument
         """
-        # get document with all attachments (u1db content and eventual
-        # conflicts)
+        doc_from_batch = self.__check_batch_before_get(doc_id)
+        if doc_from_batch:
+            return doc_from_batch
         if self.batching and doc_id not in self.batched_ids:
             return None
         if doc_id not in self._database:
             return None
+        # get document with all attachments (u1db content and eventual
+        # conflicts)
         result = self.json_from_resource([doc_id], attachments=True)
         return self.__parse_doc_from_couch(result, doc_id, check_for_conflicts)
 
+    def __check_batch_before_get(self, doc_id):
+        """
+        If doc_id is staged for batching, then we need to commit the batch
+        before going ahead. This avoids consistency problems, like trying to
+        get a document that isn't persisted and processing like it is missing.
+
+        :param doc_id: The unique document identifier
+        :type doc_id: str
+        """
+        if doc_id in self.batch_docs:
+            couch_doc = self.batch_docs[doc_id]
+            rev = self.__perform_batch(doc_id)
+            couch_doc['_rev'] = rev
+            self.batched_ids.add(doc_id)
+            return self.__parse_doc_from_couch(couch_doc, doc_id, True)
+        return None
+
+    def __perform_batch(self, doc_id=None):
+        status = self._database.update(self.batch_docs.values())
+        rev = None
+        for ok, stored_doc_id, rev_or_error in status:
+            if not ok:
+                error = rev_or_error
+                if type(error) is ResourceConflict:
+                    raise RevisionConflict
+                raise error
+            elif doc_id == stored_doc_id:
+                rev = rev_or_error
+        self.batch_docs.clear()
+        return rev
+
     def __parse_doc_from_couch(self, result, doc_id,
                                check_for_conflicts=False):
         # restrict to u1db documents
@@ -726,8 +760,8 @@ class CouchDatabase(object):
                 del attachment['length']
                 index = 0 if name is 'u1db_content' else 1
                 attachment['data'] = binascii.b2a_base64(parts[index]).strip()
-            couch_doc['attachments'] = attachments
-            self.batch_docs.append(couch_doc)
+            couch_doc['_attachments'] = attachments
+            self.batch_docs[doc.doc_id] = couch_doc
         return transactions[-1][1]
 
     def _new_resource(self, *path):
-- 
cgit v1.2.3


From f1497b92aff3b953eca572c08d85d8ddffb36391 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 17 Nov 2015 21:27:38 -0300
Subject: [feat] add configuration to disable batching

Batch support is optional. This commit adds a 'batching' configuration
option to disable it.
---
 common/src/leap/soledad/common/backend.py          |  5 ++++
 .../leap/soledad/common/tests/fixture_soledad.conf |  1 +
 .../src/leap/soledad/common/tests/test_server.py   | 33 +++++++++++++---------
 server/src/leap/soledad/server/__init__.py         | 25 +++++++++-------
 4 files changed, 41 insertions(+), 23 deletions(-)

diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
index 9f5950b2..91f28fff 100644
--- a/common/src/leap/soledad/common/backend.py
+++ b/common/src/leap/soledad/common/backend.py
@@ -33,6 +33,7 @@ from leap.soledad.common.document import ServerDocument
 
 
 class SoledadBackend(CommonBackend):
+    BATCH_SUPPORT = True
 
     """
     A U1DB backend implementation.
@@ -58,6 +59,8 @@ class SoledadBackend(CommonBackend):
             self._set_replica_uid(replica_uid)
 
     def batch_start(self):
+        if not self.BATCH_SUPPORT:
+            return
         self.batching = True
         self.after_batch_callbacks = {}
         self._database.batch_start()
@@ -67,6 +70,8 @@ class SoledadBackend(CommonBackend):
         self._get_generation()  # warm up gen info
 
     def batch_end(self):
+        if not self.BATCH_SUPPORT:
+            return
         self.batching = False
         self._database.batch_end()
         for name in self.after_batch_callbacks:
diff --git a/common/src/leap/soledad/common/tests/fixture_soledad.conf b/common/src/leap/soledad/common/tests/fixture_soledad.conf
index c0ffacf6..8d8161c3 100644
--- a/common/src/leap/soledad/common/tests/fixture_soledad.conf
+++ b/common/src/leap/soledad/common/tests/fixture_soledad.conf
@@ -2,6 +2,7 @@
 couch_url   = http://soledad:passwd@localhost:5984
 create_cmd  = sudo -u soledad-admin /usr/bin/create-user-db
 admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc
+batching    = 0
 
 [database-security]
 members       = user1, user2
diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py
index e1129a9f..20fe8579 100644
--- a/common/src/leap/soledad/common/tests/test_server.py
+++ b/common/src/leap/soledad/common/tests/test_server.py
@@ -640,18 +640,25 @@ class ConfigurationParsingTest(unittest.TestCase):
         config = load_configuration(config_path)
 
         # then
-        expected = {'soledad-server': {
-                    'couch_url':
-                        'http://soledad:passwd@localhost:5984',
-                    'create_cmd':
-                        'sudo -u soledad-admin /usr/bin/create-user-db',
-                    'admin_netrc':
-                        '/etc/couchdb/couchdb-soledad-admin.netrc',
-                    },
-                    'database-security': {
-                    'members': ['user1', 'user2'],
+        expected = {'members': ['user1', 'user2'],
                     'members_roles': ['role1', 'role2'],
                     'admins': ['user3', 'user4'],
-                    'admins_roles': ['role3', 'role3'],
-        }}
-        self.assertDictEqual(expected, config)
+                    'admins_roles': ['role3', 'role3']}
+        self.assertDictEqual(expected, config['database-security'])
+
+    def test_server_values_configuration(self):
+        # given
+        config_path = resource_filename('leap.soledad.common.tests',
+                                        'fixture_soledad.conf')
+        # when
+        config = load_configuration(config_path)
+
+        # then
+        expected = {'couch_url':
+                    'http://soledad:passwd@localhost:5984',
+                    'create_cmd':
+                    'sudo -u soledad-admin /usr/bin/create-user-db',
+                    'admin_netrc':
+                    '/etc/couchdb/couchdb-soledad-admin.netrc',
+                    'batching': False}
+        self.assertDictEqual(expected, config['soledad-server'])
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 00e1e9fb..7320c133 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -104,6 +104,7 @@ from leap.soledad.server.sync import (
 )
 
 from leap.soledad.common import SHARED_DB_NAME
+from leap.soledad.common.backend import SoledadBackend
 from leap.soledad.common.couch.state import CouchServerState
 
 # ----------------------------------------------------------------------------
@@ -264,6 +265,7 @@ CONFIG_DEFAULTS = {
         'couch_url': 'http://localhost:5984',
         'create_cmd': None,
         'admin_netrc': '/etc/couchdb/couchdb-admin.netrc',
+        'batching': True
     },
     'database-security': {
         'members': ['soledad'],
@@ -285,18 +287,20 @@ def load_configuration(file_path):
     @rtype: dict
     """
     defaults = dict(CONFIG_DEFAULTS)
-    config = configparser.ConfigParser()
+    config = configparser.SafeConfigParser()
     config.read(file_path)
-    for section in defaults.keys():
-        if section in config:
-            for key in defaults[section]:
-                if key in config[section]:
-                    defaults[section][key] = config[section][key]
-    for key, value in defaults['database-security'].iteritems():
-        if type(value) is not unicode:
+    for section in defaults:
+        if not config.has_section(section):
             continue
-        defaults['database-security'][key] = \
-                [item.strip() for item in value.split(',')]
+        for key, value in defaults[section].items():
+            if type(value) == bool:
+                defaults[section][key] = config.getboolean(section, key)
+            elif type(value) == list:
+                values = config.get(section, key).split(',')
+                values = [v.strip() for v in values]
+                defaults[section][key] = values
+            else:
+                defaults[section][key] = config.get(section, key)
     # TODO: implement basic parsing/sanitization of options comming from
     # config file.
     return defaults
@@ -310,6 +314,7 @@ def application(environ, start_response):
     conf = load_configuration('/etc/soledad/soledad-server.conf')
     conf = conf['soledad-server']
     state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd'])
+    SoledadBackend.BATCH_SUPPORT = conf['batching']
     # WSGI application that may be used by `twistd -web`
     application = GzipMiddleware(
         SoledadTokenAuthMiddleware(SoledadApp(state)))
-- 
cgit v1.2.3


From d103491cbc17e6e7422653e9b01101ff446e7391 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Thu, 3 Dec 2015 19:25:29 -0300
Subject: [feat] generation caching during a batch

Generation cache was removed for simple processing and it should not got
back, but during a batch the server wont change its generation. So a
little trick to hold that temporary information until batch finishes is
needed.
---
 common/src/leap/soledad/common/couch/__init__.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
index a922fd48..dae460cb 100644
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -154,6 +154,7 @@ class CouchDatabase(object):
         self._dbname = dbname
         self._database = self.get_couch_database(url, dbname)
         self.batching = False
+        self.batch_generation = None
         self.batch_docs = {}
         if ensure_ddocs:
             self.ensure_ddocs_on_db()
@@ -161,11 +162,13 @@ class CouchDatabase(object):
 
     def batch_start(self):
         self.batching = True
+        self.batch_generation = self.get_generation_info()
         ids = set(row.id for row in self._database.view('_all_docs'))
         self.batched_ids = ids
 
     def batch_end(self):
         self.batching = False
+        self.batch_generation = None
         self.__perform_batch()
 
     def get_couch_database(self, url, dbname):
@@ -619,6 +622,8 @@ class CouchDatabase(object):
         :return: A tuple containing the current generation and transaction id.
         :rtype: (int, str)
         """
+        if self.batching and self.batch_generation:
+            return self.batch_generation
         # query a couch list function
         ddoc_path = ['_design', 'transactions', '_list', 'generation', 'log']
         info = self.json_from_resource(ddoc_path)
@@ -762,6 +767,8 @@ class CouchDatabase(object):
                 attachment['data'] = binascii.b2a_base64(parts[index]).strip()
             couch_doc['_attachments'] = attachments
             self.batch_docs[doc.doc_id] = couch_doc
+            last_gen, last_trans_id = self.batch_generation
+            self.batch_generation = (last_gen + 1, transaction_id)
         return transactions[-1][1]
 
     def _new_resource(self, *path):
-- 
cgit v1.2.3


From 7208d8bc5e5f23d0773533b15763f64d236489b4 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Thu, 3 Dec 2015 19:34:56 -0300
Subject: [feat] set default to False on batching for now

All batching code has no effect by default with this commit. Since we
know that this is a dangerous new feature we will enable them only on
our test servers and check them manually before setting it as default
or adding more configuration features.
Use SyncTarget and server conf file to enable it for testing.
---
 client/src/leap/soledad/client/http_target/send.py | 2 +-
 common/src/leap/soledad/common/backend.py          | 2 +-
 server/src/leap/soledad/server/__init__.py         | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index c1252c13..e8abf35b 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -29,7 +29,7 @@ class HTTPDocSender(object):
     They need to be encrypted and metadata prepared before sending.
     """
 
-    MAX_BATCH_SIZE = 500 * 1000  # 500kB by default
+    MAX_BATCH_SIZE = 0  # disabled by now, this is being tested yet
 
     @defer.inlineCallbacks
     def _send_docs(self, docs_by_generation, last_known_generation,
diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py
index 91f28fff..53426fb5 100644
--- a/common/src/leap/soledad/common/backend.py
+++ b/common/src/leap/soledad/common/backend.py
@@ -33,7 +33,7 @@ from leap.soledad.common.document import ServerDocument
 
 
 class SoledadBackend(CommonBackend):
-    BATCH_SUPPORT = True
+    BATCH_SUPPORT = False
 
     """
     A U1DB backend implementation.
diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 7320c133..39edcc1b 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -265,7 +265,7 @@ CONFIG_DEFAULTS = {
         'couch_url': 'http://localhost:5984',
         'create_cmd': None,
         'admin_netrc': '/etc/couchdb/couchdb-admin.netrc',
-        'batching': True
+        'batching': False
     },
     'database-security': {
         'members': ['soledad'],
-- 
cgit v1.2.3


From 27bda0ac201e236e3a2c9671462a337f2970e993 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Fri, 4 Dec 2015 15:47:53 -0300
Subject: [bug] skip missing keys on existing sections

While parsing the configuration file, if a key doesnt exist we need to
skip it.
---
 server/src/leap/soledad/server/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py
index 39edcc1b..22894dac 100644
--- a/server/src/leap/soledad/server/__init__.py
+++ b/server/src/leap/soledad/server/__init__.py
@@ -293,7 +293,9 @@ def load_configuration(file_path):
         if not config.has_section(section):
             continue
         for key, value in defaults[section].items():
-            if type(value) == bool:
+            if not config.has_option(section, key):
+                continue
+            elif type(value) == bool:
                 defaults[section][key] = config.getboolean(section, key)
             elif type(value) == list:
                 values = config.get(section, key).split(',')
-- 
cgit v1.2.3


From 2ed8f2b64f3093255d03f282917eb88aac9ab124 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 30 Nov 2015 12:21:56 -0400
Subject: [feat] use uuid in events, multi-user aware

- Resolves: #7656
- Releases: 0.8.0
---
 client/changes/next-changelog.rst                  | 29 ++++++++++++++++++++++
 client/src/leap/soledad/client/api.py              |  6 ++---
 .../leap/soledad/client/http_target/__init__.py    |  1 +
 client/src/leap/soledad/client/http_target/api.py  |  5 ++++
 .../src/leap/soledad/client/http_target/fetch.py   | 12 ++++++---
 client/src/leap/soledad/client/http_target/send.py | 13 +++++++---
 6 files changed, 57 insertions(+), 9 deletions(-)
 create mode 100644 client/changes/next-changelog.rst

diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst
new file mode 100644
index 00000000..4d527568
--- /dev/null
+++ b/client/changes/next-changelog.rst
@@ -0,0 +1,29 @@
+0.8.0 - xxx
++++++++++++++++++++++++++++++++
+
+Please add lines to this file, they will be moved to the CHANGELOG.rst during
+the next release.
+
+There are two template lines for each category, use them as reference.
+
+I've added a new category `Misc` so we can track doc/style/packaging stuff.
+
+Features
+~~~~~~~~
+- `#7656 <https://leap.se/code/issues/7656>`_: Emit multi-user aware events.
+- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
+- New feature without related issue number.
+
+Bugfixes
+~~~~~~~~
+- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235.
+- Bugfix without related issue number.
+
+Misc
+~~~~
+- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
+- Some change without issue number.
+
+Known Issues
+~~~~~~~~~~~~
+- `#1236 <https://leap.se/code/issues/1236>`_: Description of the known issue corresponding with issue #1236.
diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 8c5f7f1b..0d0d636c 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -102,11 +102,11 @@ class Soledad(object):
             soledad starts to retrieve keys from server.
         SOLEDAD_DONE_DOWNLOADING_KEYS: emitted during bootstrap sequence when
             soledad finishes downloading keys from server.
-        SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when
-          there's indeed new data to be synchronized between local database
-          replica and server's replica.
         SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has
             finished synchronizing with remote replica.
+        SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when
+          there's indeed new data to be synchronized between local database
+          replica and server's replica. --- not used right now.
     """
     implements(soledad_interfaces.ILocalStorage,
                soledad_interfaces.ISyncableStorage,
diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py
index 498fb6e7..a16531ef 100644
--- a/client/src/leap/soledad/client/http_target/__init__.py
+++ b/client/src/leap/soledad/client/http_target/__init__.py
@@ -79,6 +79,7 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher):
         self._url = str(url) + "/sync-from/" + str(source_replica_uid)
         self.source_replica_uid = source_replica_uid
         self._auth_header = None
+        self._uuid = None
         self.set_creds(creds)
         self._crypto = crypto
         self._sync_db = sync_db
diff --git a/client/src/leap/soledad/client/http_target/api.py b/client/src/leap/soledad/client/http_target/api.py
index dcc762f6..94354092 100644
--- a/client/src/leap/soledad/client/http_target/api.py
+++ b/client/src/leap/soledad/client/http_target/api.py
@@ -40,6 +40,10 @@ class SyncTargetAPI(SyncTarget):
             self._sync_decr_pool.stop()
         yield self._http.close()
 
+    @property
+    def uuid(self):
+        return self._uuid
+
     def set_creds(self, creds):
         """
         Update credentials.
@@ -49,6 +53,7 @@ class SyncTargetAPI(SyncTarget):
         """
         uuid = creds['token']['uuid']
         token = creds['token']['token']
+        self._uuid = uuid
         auth = '%s:%s' % (uuid, token)
         b64_token = base64.b64encode(auth)
         self._auth_header = {'Authorization': ['Token %s' % b64_token]}
diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py
index 65e576d9..087da3a3 100644
--- a/client/src/leap/soledad/client/http_target/fetch.py
+++ b/client/src/leap/soledad/client/http_target/fetch.py
@@ -39,6 +39,12 @@ class HTTPDocFetcher(object):
               So we parse, decrypt and insert locally as they arrive.
     """
 
+    # The uuid of the local replica.
+    # Any class inheriting from this one should provide a meaningful attribute
+    # if the sync status event is meant to be used somewhere else.
+
+    uuid = 'undefined'
+
     @defer.inlineCallbacks
     def _receive_docs(self, last_known_generation, last_known_trans_id,
                       ensure_callback, sync_id, defer_decryption):
@@ -176,7 +182,7 @@ class HTTPDocFetcher(object):
             # end of symmetric decryption
             # -------------------------------------------------------------
         self._received_docs += 1
-        _emit_receive_status(self._received_docs, total)
+        _emit_receive_status(self.uuid, self._received_docs, total)
         return number_of_changes, new_generation, new_transaction_id
 
     def _parse_received_doc_response(self, response):
@@ -243,9 +249,9 @@ class HTTPDocFetcher(object):
                 source_replica_uid=self.source_replica_uid)
 
 
-def _emit_receive_status(received_docs, total):
+def _emit_receive_status(uuid, received_docs, total):
     content = {'received': received_docs, 'total': total}
-    emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, content)
+    emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, uuid, content)
 
     if received_docs % 20 == 0:
         msg = "%d/%d" % (received_docs, total)
diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index e8abf35b..3fc5a171 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -31,6 +31,12 @@ class HTTPDocSender(object):
 
     MAX_BATCH_SIZE = 0  # disabled by now, this is being tested yet
 
+    # The uuid of the local replica.
+    # Any class inheriting from this one should provide a meaningful attribute
+    # if the sync status event is meant to be used somewhere else.
+
+    uuid = 'undefined'
+
     @defer.inlineCallbacks
     def _send_docs(self, docs_by_generation, last_known_generation,
                    last_known_trans_id, sync_id):
@@ -71,7 +77,8 @@ class HTTPDocSender(object):
         result = yield self._send_request(body.pop())
         if self._defer_encryption:
             self._delete_sent(sent)
-        _emit_send_status(body.consumed, total)
+
+        _emit_send_status(self.uuid, body.consumed, total)
         defer.returnValue(result)
 
     def _send_request(self, body):
@@ -112,9 +119,9 @@ class HTTPDocSender(object):
         return d
 
 
-def _emit_send_status(idx, total):
+def _emit_send_status(uuid, idx, total):
     content = {'sent': idx, 'total': total}
-    emit_async(SOLEDAD_SYNC_SEND_STATUS, content)
+    emit_async(SOLEDAD_SYNC_SEND_STATUS, uuid, content)
 
     msg = "%d/%d" % (idx, total)
     logger.debug("Sync send status: %s" % msg)
-- 
cgit v1.2.3


From 402e00bbd3dd98363ce7ffa31cb9aeae1490095e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 7 Dec 2015 12:06:51 -0400
Subject: [feat] use userid in soledad events too

for the moment, userid has to be passed to constructor.
eventually, we might drop support for passing uuid, since it will be
mapped in the service tree
---
 client/src/leap/soledad/client/api.py               | 11 +++++++++--
 client/src/leap/soledad/client/http_target/fetch.py |  8 +++++---
 client/src/leap/soledad/client/http_target/send.py  |  6 ++++--
 client/src/leap/soledad/client/secrets.py           | 19 +++++++++++++------
 4 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 0d0d636c..02d9c735 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -125,7 +125,8 @@ class Soledad(object):
 
     def __init__(self, uuid, passphrase, secrets_path, local_db_path,
                  server_url, cert_file, shared_db=None,
-                 auth_token=None, defer_encryption=False, syncable=True):
+                 auth_token=None, defer_encryption=False, syncable=True,
+                 userid=None):
         """
         Initialize configuration, cryptographic keys and dbs.
 
@@ -179,6 +180,7 @@ class Soledad(object):
         """
         # store config params
         self._uuid = uuid
+        self._userid = userid
         self._passphrase = passphrase
         self._local_db_path = local_db_path
         self._server_url = server_url
@@ -648,6 +650,10 @@ class Soledad(object):
     def uuid(self):
         return self._uuid
 
+    @property
+    def userid(self):
+        return self._userid
+
     #
     # ISyncableStorage
     #
@@ -718,8 +724,9 @@ class Soledad(object):
             return failure
 
         def _emit_done_data_sync(passthrough):
+            user_data = {'uuid': self.uuid, 'userid': self.userid}
             soledad_events.emit_async(
-                soledad_events.SOLEDAD_DONE_DATA_SYNC, self.uuid)
+                soledad_events.SOLEDAD_DONE_DATA_SYNC, user_data)
             return passthrough
 
         d.addCallbacks(_sync_callback, _sync_errback)
diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py
index 087da3a3..9f7a4193 100644
--- a/client/src/leap/soledad/client/http_target/fetch.py
+++ b/client/src/leap/soledad/client/http_target/fetch.py
@@ -44,6 +44,7 @@ class HTTPDocFetcher(object):
     # if the sync status event is meant to be used somewhere else.
 
     uuid = 'undefined'
+    userid = 'undefined'
 
     @defer.inlineCallbacks
     def _receive_docs(self, last_known_generation, last_known_trans_id,
@@ -182,7 +183,8 @@ class HTTPDocFetcher(object):
             # end of symmetric decryption
             # -------------------------------------------------------------
         self._received_docs += 1
-        _emit_receive_status(self.uuid, self._received_docs, total)
+        user_data = {'uuid': self.uuid, 'userid': self.userid}
+        _emit_receive_status(user_data, self._received_docs, total)
         return number_of_changes, new_generation, new_transaction_id
 
     def _parse_received_doc_response(self, response):
@@ -249,9 +251,9 @@ class HTTPDocFetcher(object):
                 source_replica_uid=self.source_replica_uid)
 
 
-def _emit_receive_status(uuid, received_docs, total):
+def _emit_receive_status(user_data, received_docs, total):
     content = {'received': received_docs, 'total': total}
-    emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, uuid, content)
+    emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, user_data, content)
 
     if received_docs % 20 == 0:
         msg = "%d/%d" % (received_docs, total)
diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py
index 3fc5a171..89288779 100644
--- a/client/src/leap/soledad/client/http_target/send.py
+++ b/client/src/leap/soledad/client/http_target/send.py
@@ -36,6 +36,7 @@ class HTTPDocSender(object):
     # if the sync status event is meant to be used somewhere else.
 
     uuid = 'undefined'
+    userid = 'undefined'
 
     @defer.inlineCallbacks
     def _send_docs(self, docs_by_generation, last_known_generation,
@@ -78,6 +79,7 @@ class HTTPDocSender(object):
         if self._defer_encryption:
             self._delete_sent(sent)
 
+        user_data = {'uuid': self.uuid, 'userid': self.userid}
         _emit_send_status(self.uuid, body.consumed, total)
         defer.returnValue(result)
 
@@ -119,9 +121,9 @@ class HTTPDocSender(object):
         return d
 
 
-def _emit_send_status(uuid, idx, total):
+def _emit_send_status(user_data, idx, total):
     content = {'sent': idx, 'total': total}
-    emit_async(SOLEDAD_SYNC_SEND_STATUS, uuid, content)
+    emit_async(SOLEDAD_SYNC_SEND_STATUS, user_data, content)
 
     msg = "%d/%d" % (idx, total)
     logger.debug("Sync send status: %s" % msg)
diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
index c3c3dff5..90e73bd5 100644
--- a/client/src/leap/soledad/client/secrets.py
+++ b/client/src/leap/soledad/client/secrets.py
@@ -147,7 +147,7 @@ class SoledadSecrets(object):
     Keys used to access storage secrets in recovery documents.
     """
 
-    def __init__(self, uuid, passphrase, secrets_path, shared_db):
+    def __init__(self, uuid, passphrase, secrets_path, shared_db, userid=None):
         """
         Initialize the secrets manager.
 
@@ -167,6 +167,7 @@ class SoledadSecrets(object):
         # param secret_id: The id of the storage secret to be used.
 
         self._uuid = uuid
+        self._userid = userid
         self._passphrase = passphrase
         self._secrets_path = secrets_path
         self._shared_db = shared_db
@@ -433,7 +434,8 @@ class SoledadSecrets(object):
         :return: a document with encrypted key material in its contents
         :rtype: document.SoledadDocument
         """
-        events.emit_async(events.SOLEDAD_DOWNLOADING_KEYS, self._uuid)
+        user_data = self._get_user_data()
+        events.emit_async(events.SOLEDAD_DOWNLOADING_KEYS, user_data)
         db = self._shared_db
         if not db:
             logger.warning('No shared db found')
@@ -462,13 +464,14 @@ class SoledadSecrets(object):
         # fill doc with encrypted secrets
         doc.content = self._export_recovery_document()
         # upload secrets to server
-        events.emit_async(events.SOLEDAD_UPLOADING_KEYS, self._uuid)
+        user_data = self._get_user_data()
+        events.emit_async(events.SOLEDAD_UPLOADING_KEYS, user_data)
         db = self._shared_db
         if not db:
             logger.warning('No shared db found')
             return
         db.put_doc(doc)
-        events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, self._uuid)
+        events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, user_data)
 
     #
     # Management of secret for symmetric encryption.
@@ -588,13 +591,14 @@ class SoledadSecrets(object):
         :return: The id of the generated secret.
         :rtype: str
         """
-        events.emit_async(events.SOLEDAD_CREATING_KEYS, self._uuid)
+        user_data = self._get_user_data()
+        events.emit_async(events.SOLEDAD_CREATING_KEYS, user_data)
         # generate random secret
         secret = os.urandom(self.GEN_SECRET_LENGTH)
         secret_id = sha256(secret).hexdigest()
         self._secrets[secret_id] = secret
         self._store_secrets()
-        events.emit_async(events.SOLEDAD_DONE_CREATING_KEYS, self._uuid)
+        events.emit_async(events.SOLEDAD_DONE_CREATING_KEYS, user_data)
         return secret_id
 
     def _store_secrets(self):
@@ -738,3 +742,6 @@ class SoledadSecrets(object):
             salt=self._get_sync_db_salt(),
             buflen=32,  # we need a key with 256 bits (32 bytes)
         )
+
+    def _get_user_data(self):
+        return {'uuid': self._uuid, 'userid': self._userid}
-- 
cgit v1.2.3


From a750a80d35b19d1d8bd410c194531b8b92c6eb96 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 14 Dec 2015 10:12:15 -0400
Subject: [bug] fix failing tests after last events modification

---
 client/src/leap/soledad/client/api.py              |  2 +-
 client/src/leap/soledad/client/secrets.py          |  3 +-
 .../src/leap/soledad/common/tests/test_soledad.py  | 36 ++++++++++------------
 common/src/leap/soledad/common/tests/util.py       |  7 +++--
 4 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 02d9c735..619671fd 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -252,7 +252,7 @@ class Soledad(object):
         """
         self._secrets = SoledadSecrets(
             self.uuid, self._passphrase, self._secrets_path,
-            self.shared_db)
+            self.shared_db, userid=self._userid)
         self._secrets.bootstrap()
 
     def _init_u1db_sqlcipher_backend(self):
diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py
index 90e73bd5..e2a5a1d7 100644
--- a/client/src/leap/soledad/client/secrets.py
+++ b/client/src/leap/soledad/client/secrets.py
@@ -441,7 +441,8 @@ class SoledadSecrets(object):
             logger.warning('No shared db found')
             return
         doc = db.get_doc(self._shared_db_doc_id())
-        events.emit_async(events.SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid)
+        user_data = {'userid': self._userid, 'uuid': self._uuid}
+        events.emit_async(events.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data)
         return doc
 
     def _put_secrets_in_shared_db(self):
diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py
index 85d6734e..a2375b6e 100644
--- a/common/src/leap/soledad/common/tests/test_soledad.py
+++ b/common/src/leap/soledad/common/tests/test_soledad.py
@@ -249,55 +249,51 @@ class SoledadSignalingTestCase(BaseSoledadTest):
         # get a fresh instance so it emits all bootstrap signals
         sol = self._soledad_instance(
             secrets_path='alternative_stage3.json',
-            local_db_path='alternative_stage3.u1db')
+            local_db_path='alternative_stage3.u1db',
+            userid=ADDRESS)
         # reverse call order so we can verify in the order the signals were
         # expected
         soledad.client.secrets.events.emit_async.mock_calls.reverse()
         soledad.client.secrets.events.emit_async.call_args = \
             soledad.client.secrets.events.emit_async.call_args_list[0]
         soledad.client.secrets.events.emit_async.call_args_list.reverse()
+
+        user_data = {'userid': ADDRESS, 'uuid': ADDRESS}
+
         # downloading keys signals
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_DOWNLOADING_KEYS,
-            ADDRESS,
+            catalog.SOLEDAD_DOWNLOADING_KEYS, user_data
         )
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_DONE_DOWNLOADING_KEYS,
-            ADDRESS,
+            catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data
         )
         # creating keys signals
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_CREATING_KEYS,
-            ADDRESS,
+            catalog.SOLEDAD_CREATING_KEYS, user_data
         )
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_DONE_CREATING_KEYS,
-            ADDRESS,
+            catalog.SOLEDAD_DONE_CREATING_KEYS, user_data
         )
         # downloading once more (inside _put_keys_in_shared_db)
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_DOWNLOADING_KEYS,
-            ADDRESS,
+            catalog.SOLEDAD_DOWNLOADING_KEYS, user_data
         )
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_DONE_DOWNLOADING_KEYS,
-            ADDRESS,
+            catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data 
         )
         # uploading keys signals
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_UPLOADING_KEYS,
-            ADDRESS,
+            catalog.SOLEDAD_UPLOADING_KEYS, user_data 
         )
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_DONE_UPLOADING_KEYS,
-            ADDRESS,
+            catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data 
         )
         # assert db was locked and unlocked
         sol.shared_db.lock.assert_called_with()
@@ -332,12 +328,12 @@ class SoledadSignalingTestCase(BaseSoledadTest):
         # assert download keys signals
         soledad.client.secrets.events.emit_async.assert_called_with(
             catalog.SOLEDAD_DOWNLOADING_KEYS,
-            ADDRESS,
+            {'userid': ADDRESS, 'uuid': ADDRESS}
         )
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
             catalog.SOLEDAD_DONE_DOWNLOADING_KEYS,
-            ADDRESS,
+            {'userid': ADDRESS, 'uuid': ADDRESS},
         )
         sol.close()
 
@@ -371,6 +367,6 @@ class SoledadSignalingTestCase(BaseSoledadTest):
         # assert the signal has been emitted
         soledad.client.events.emit_async.assert_called_with(
             catalog.SOLEDAD_DONE_DATA_SYNC,
-            ADDRESS,
+            {'userid': ADDRESS, 'uuid': ADDRESS},
         )
         sol.close()
diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py
index f7f9ebd0..d4510686 100644
--- a/common/src/leap/soledad/common/tests/util.py
+++ b/common/src/leap/soledad/common/tests/util.py
@@ -245,6 +245,7 @@ class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest):
         # each local db.
         self.rand_prefix = ''.join(
             map(lambda x: random.choice(string.ascii_letters), range(6)))
+
         # initialize soledad by hand so we can control keys
         # XXX check if this soledad is actually used
         self._soledad = self._soledad_instance(
@@ -285,7 +286,8 @@ class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest):
                           server_url='https://127.0.0.1/',
                           cert_file=None,
                           shared_db_class=None,
-                          auth_token='auth-token'):
+                          auth_token='auth-token',
+                          userid=ADDRESS):
 
         def _put_doc_side_effect(doc):
             self._doc_put = doc
@@ -307,7 +309,8 @@ class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest):
             cert_file=cert_file,
             defer_encryption=self.defer_sync_encryption,
             shared_db=MockSharedDB(),
-            auth_token=auth_token)
+            auth_token=auth_token,
+            userid=userid)
         self.addCleanup(soledad.close)
         return soledad
 
-- 
cgit v1.2.3


From def5b89e6066335708b93b1dda1e65aaca8598ac Mon Sep 17 00:00:00 2001
From: Christoph Kluenter <ckluente@thouthworks.com>
Date: Mon, 14 Dec 2015 17:19:48 +0100
Subject: [fix] remove trailing whitespace to please pep8

---
 common/src/leap/soledad/common/tests/test_soledad.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py
index a2375b6e..36c4003c 100644
--- a/common/src/leap/soledad/common/tests/test_soledad.py
+++ b/common/src/leap/soledad/common/tests/test_soledad.py
@@ -284,16 +284,16 @@ class SoledadSignalingTestCase(BaseSoledadTest):
         )
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data 
+            catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data
         )
         # uploading keys signals
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_UPLOADING_KEYS, user_data 
+            catalog.SOLEDAD_UPLOADING_KEYS, user_data
         )
         self._pop_mock_call(soledad.client.secrets.events.emit_async)
         soledad.client.secrets.events.emit_async.assert_called_with(
-            catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data 
+            catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data
         )
         # assert db was locked and unlocked
         sol.shared_db.lock.assert_called_with()
-- 
cgit v1.2.3


From 869aa5354ff74be9e0203c37c5d54c79d8e32863 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 26 Nov 2015 22:13:34 -0400
Subject: [feat] set syncable attribute

this allows to switch the online/offline mode on a running soledad
instance.
---
 client/src/leap/soledad/client/api.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 619671fd..78f9c652 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -187,6 +187,7 @@ class Soledad(object):
         self._defer_encryption = defer_encryption
         self._secrets_path = None
         self._sync_enc_pool = None
+        self._dbsyncer = None
 
         self.shared_db = shared_db
 
@@ -217,6 +218,7 @@ class Soledad(object):
     #
     # initialization/destruction methods
     #
+
     def _init_config_with_defaults(self):
         """
         Initialize configuration using default values for missing params.
@@ -658,6 +660,21 @@ class Soledad(object):
     # ISyncableStorage
     #
 
+    def set_syncable(self, syncable):
+        """
+        Toggle the syncable state for this database.
+
+        This can be used to start a database with offline state and switch it
+        online afterwards. Or the opposite: stop syncs when connection is lost.
+
+        :param syncable: new status for syncable.
+        :type syncable: bool
+        """
+        # TODO should check that we've got a token!
+        self.shared_db.syncable = syncable
+        if syncable and not self._dbsyncer:
+            self._init_u1db_syncer()
+
     def sync(self, defer_decryption=True):
         """
         Synchronize documents with the server replica.
@@ -754,6 +771,14 @@ class Soledad(object):
         """
         return self.sync_lock.locked
 
+    @property
+    def syncable(self):
+        if self.shared_db:
+            return self.shared_db.syncable
+        else:
+            return False
+
+
     def _set_token(self, token):
         """
         Set the authentication token for remote database access.
-- 
cgit v1.2.3


From 0d7470b48411f1b9d48762e33533bb9041676c2e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 2 Dec 2015 15:09:58 -0400
Subject: [feat] get_or_create_service_token

---
 client/src/leap/soledad/client/api.py | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 78f9c652..1f81f2ab 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -32,6 +32,7 @@ import logging
 import os
 import socket
 import ssl
+import uuid
 import urlparse
 
 try:
@@ -44,11 +45,11 @@ from StringIO import StringIO
 from collections import defaultdict
 from u1db.remote import http_client
 from u1db.remote.ssl_match_hostname import match_hostname
+from twisted.internet.defer import DeferredLock, returnValue, inlineCallbacks
 from zope.interface import implements
 
 from leap.common.config import get_path_prefix
 from leap.common.plugins import collect_plugins
-from twisted.internet.defer import DeferredLock
 
 from leap.soledad.common import SHARED_DB_NAME
 from leap.soledad.common import soledad_assert
@@ -943,6 +944,38 @@ class Soledad(object):
         """
         return self._dbpool.runOperation(*args, **kw)
 
+    #
+    # Service authentication
+    #
+
+    @inlineCallbacks
+    def get_or_create_service_token(self, service):
+        """
+        Return the stored token for a given service, or generates and stores a
+        random one if it does not exist.
+
+        These tokens can be used to authenticate services.
+        """
+        # FIXME this could use the local sqlcipher database, to avoid
+        # problems with different replicas creating different tokens.
+
+        yield self.create_index('by-servicetoken', 'type', 'service')
+        docs = yield self._get_token_for_service(service)
+        if docs:
+            doc = docs[0]
+            returnValue(doc.content['token'])
+        else:
+            token = str(uuid.uuid4()).replace('-', '')[-24:]
+            yield self._set_token_for_service(service, token)
+            returnValue(token)
+
+    def _get_token_for_service(self, service):
+        return self.get_from_index('by-servicetoken', 'servicetoken', service)
+
+    def _set_token_for_service(self, service, token):
+        doc = {'type': 'servicetoken', 'service': service, 'token': token}
+        return self.create_doc(doc)
+
 
 def _convert_to_unicode(content):
     """
-- 
cgit v1.2.3


From da58f335596c856dee21e2ad014717912e4206d3 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Tue, 15 Dec 2015 16:44:59 -0400
Subject: [style] pep8

---
 client/src/leap/soledad/client/api.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py
index 1f81f2ab..e657c939 100644
--- a/client/src/leap/soledad/client/api.py
+++ b/client/src/leap/soledad/client/api.py
@@ -779,7 +779,6 @@ class Soledad(object):
         else:
             return False
 
-
     def _set_token(self, token):
         """
         Set the authentication token for remote database access.
-- 
cgit v1.2.3


From 888d3d532770cb06fbc575088dc3e27edff8d703 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 22 Dec 2015 19:50:48 -0300
Subject: [docs] incomplete doc for security config parameter

database_security parameter was either undocumented or incomplete. This
commit adds a few more doc to make it consistent with latest changes.
Closes #7689
---
 common/src/leap/soledad/common/couch/__init__.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py
index dae460cb..18ed8a19 100644
--- a/common/src/leap/soledad/common/couch/__init__.py
+++ b/common/src/leap/soledad/common/couch/__init__.py
@@ -123,6 +123,8 @@ class CouchDatabase(object):
         :type replica_uid: str
         :param ensure_ddocs: Ensure that the design docs exist on server.
         :type ensure_ddocs: bool
+        :param database_security: security rules as CouchDB security doc
+        :type database_security: dict
 
         :return: the database instance
         :rtype: SoledadBackend
@@ -149,6 +151,16 @@ class CouchDatabase(object):
 
     def __init__(self, url, dbname, ensure_ddocs=True,
                  database_security=None):
+        """
+        :param url: Couch server URL with necessary credentials
+        :type url: string
+        :param dbname: Couch database name
+        :type dbname: string
+        :param ensure_ddocs: Ensure that the design docs exist on server.
+        :type ensure_ddocs: bool
+        :param database_security: security rules as CouchDB security doc
+        :type database_security: dict
+        """
         self._session = Session(timeout=COUCH_TIMEOUT)
         self._url = url
         self._dbname = dbname
@@ -218,8 +230,8 @@ class CouchDatabase(object):
         This is achieved by creating a _security design document, see:
         http://docs.couchdb.org/en/latest/api/database/security.html
 
-        :param database_security: security configuration parsed from conf file
-        :type cache: dict
+        :param security_config: security configuration parsed from conf file
+        :type security_config: dict
         """
         security_config = security_config or {}
         security = self._database.resource.get_json('_security')[2]
-- 
cgit v1.2.3


From 4c214b7ff9351dd98efbb033dd5e09b8ff9ff763 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 20 Jan 2016 07:54:35 -0300
Subject: [Fix] slow IO-bound calls block reactor

- Move them to a thread so reactor can continue
  processing e.g. http requests
---
 client/src/leap/soledad/client/encdecpool.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py
index 0954c1df..34667a1e 100644
--- a/client/src/leap/soledad/client/encdecpool.py
+++ b/client/src/leap/soledad/client/encdecpool.py
@@ -28,6 +28,7 @@ import json
 import logging
 
 from twisted.internet import reactor
+from twisted.internet import threads
 from twisted.internet import defer
 from twisted.python import log
 
@@ -687,7 +688,11 @@ class SyncDecrypterPool(SyncEncryptDecryptPool):
         """
         insertable = yield self._get_insertable_docs()
         for doc_fields in insertable:
-            self._insert_decrypted_local_doc(*doc_fields)
+            method = self._insert_decrypted_local_doc
+            # FIXME: This is used only because SQLCipherU1DBSync is synchronous
+            # When adbapi is used there is no need for an external thread
+            # Without this the reactor can freeze and fail docs download
+            yield threads.deferToThread(method, *doc_fields)
         defer.returnValue(insertable)
 
     def _delete_processed_docs(self, inserted):
-- 
cgit v1.2.3


From 916eaecea19f611106c2b4cea1140025c9f87ba3 Mon Sep 17 00:00:00 2001
From: Folker Bernitt <fbernitt@thoughtworks.com>
Date: Thu, 21 Jan 2016 11:44:27 +0100
Subject: [Fix] fix concurrency problem in test_sync_deferred

- Use dbsyncer (SQLCipherU1DBSync) instead of SQLCipherDatabase
  as only the first one supports multiple threads while syncing
  and is actually used by Soledad.sync
---
 common/src/leap/soledad/common/tests/test_sync_deferred.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/common/src/leap/soledad/common/tests/test_sync_deferred.py b/common/src/leap/soledad/common/tests/test_sync_deferred.py
index 90b00670..c62bd156 100644
--- a/common/src/leap/soledad/common/tests/test_sync_deferred.py
+++ b/common/src/leap/soledad/common/tests/test_sync_deferred.py
@@ -148,6 +148,8 @@ class TestSoledadDbSyncDeferredEncDecr(
         replica_uid = self._soledad._dbpool.replica_uid
         sync_db = self._soledad._sync_db
         sync_enc_pool = self._soledad._sync_enc_pool
+        dbsyncer = self._soledad._dbsyncer  # Soledad.sync uses the dbsyncer
+
         target = soledad_sync_target(
             self, self.db2._dbname,
             source_replica_uid=replica_uid,
@@ -155,7 +157,7 @@ class TestSoledadDbSyncDeferredEncDecr(
             sync_enc_pool=sync_enc_pool)
         self.addCleanup(target.close)
         return sync.SoledadSynchronizer(
-            self.db1,
+            dbsyncer,
             target).sync(defer_decryption=True)
 
     def wait_for_sync(self):
-- 
cgit v1.2.3


From 456e6911445dd255580ff023f12e1db3c6a91f7d Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 26 Jan 2016 16:05:20 -0300
Subject: [feat] use cryptography instead of pycryptopp

cryptography comes from OpenSSL and Twisted dependencies, so it's
already installed.
This commit removes a compiled dependency, also possibly making it
easier to use on Windows.
---
 client/changes/feat_use_cryptography     |  1 +
 client/pkg/requirements.pip              |  1 -
 client/src/leap/soledad/client/crypto.py | 15 +++++++++++----
 3 files changed, 12 insertions(+), 5 deletions(-)
 create mode 100644 client/changes/feat_use_cryptography

diff --git a/client/changes/feat_use_cryptography b/client/changes/feat_use_cryptography
new file mode 100644
index 00000000..6e8fe3bf
--- /dev/null
+++ b/client/changes/feat_use_cryptography
@@ -0,0 +1 @@
+o Use cryptography instead of pycryptopp. Stick with AES-CTR.
diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip
index f29d5c74..2f658d76 100644
--- a/client/pkg/requirements.pip
+++ b/client/pkg/requirements.pip
@@ -1,7 +1,6 @@
 pysqlcipher>2.6.3
 u1db
 scrypt
-pycryptopp
 cchardet
 zope.proxy
 twisted
diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 90ad656e..07a3eaab 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -24,7 +24,8 @@ import hashlib
 import json
 import logging
 
-from pycryptopp.cipher.aes import AES
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.backends import default_backend
 
 from leap.soledad.common import soledad_assert
 from leap.soledad.common import soledad_assert_type
@@ -56,7 +57,10 @@ def encrypt_sym(data, key):
         (len(key) * 8))
 
     iv = os.urandom(16)
-    ciphertext = AES(key=key, iv=iv).process(data)
+    backend = default_backend()
+    cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
+    encryptor = cipher.encryptor()
+    ciphertext = encryptor.update(data) + encryptor.finalize()
 
     return binascii.b2a_base64(iv), ciphertext
 
@@ -81,8 +85,11 @@ def decrypt_sym(data, key, iv):
     soledad_assert(
         len(key) == 32,  # 32 x 8 = 256 bits.
         'Wrong key size: %s (must be 256 bits long).' % len(key))
-    return AES(
-        key=key, iv=binascii.a2b_base64(iv)).process(data)
+    backend = default_backend()
+    iv = binascii.a2b_base64(iv)
+    cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
+    decryptor = cipher.decryptor()
+    return decryptor.update(data) + decryptor.finalize()
 
 
 def doc_mac_key(doc_id, secret):
-- 
cgit v1.2.3


From 8bf307437e0ef469bf2a5d58982a29b3c32354e2 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Thu, 28 Jan 2016 12:02:54 -0400
Subject: [docs] document deprecation of pycryptopp in changelog

---
 client/changes/next-changelog.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst
index 4d527568..89234cb2 100644
--- a/client/changes/next-changelog.rst
+++ b/client/changes/next-changelog.rst
@@ -21,6 +21,7 @@ Bugfixes
 
 Misc
 ~~~~
+- `#7195 <https://leap.se/code/issues/7195>`_: Use cryptography instead of pycryptopp.
 - `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
 - Some change without issue number.
 
-- 
cgit v1.2.3


From 45da47b7b5cddfe40820a8589d82527c2629adae Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Wed, 9 Mar 2016 12:50:08 -0400
Subject: [bug] specify openssl backend explicitely

for some reason, available_backends does not work inside a frozen
PyInstaller binary.

- Resolves: #7952
---
 client/src/leap/soledad/client/crypto.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py
index 07a3eaab..363d71b9 100644
--- a/client/src/leap/soledad/client/crypto.py
+++ b/client/src/leap/soledad/client/crypto.py
@@ -25,7 +25,8 @@ import json
 import logging
 
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
-from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.backends.multibackend import MultiBackend
+from cryptography.hazmat.backends.openssl.backend import Backend as OpenSSLBackend
 
 from leap.soledad.common import soledad_assert
 from leap.soledad.common import soledad_assert_type
@@ -57,7 +58,7 @@ def encrypt_sym(data, key):
         (len(key) * 8))
 
     iv = os.urandom(16)
-    backend = default_backend()
+    backend = MultiBackend([OpenSSLBackend()])
     cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
     encryptor = cipher.encryptor()
     ciphertext = encryptor.update(data) + encryptor.finalize()
@@ -85,7 +86,7 @@ def decrypt_sym(data, key, iv):
     soledad_assert(
         len(key) == 32,  # 32 x 8 = 256 bits.
         'Wrong key size: %s (must be 256 bits long).' % len(key))
-    backend = default_backend()
+    backend = MultiBackend([OpenSSLBackend()])
     iv = binascii.a2b_base64(iv)
     cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend)
     decryptor = cipher.decryptor()
-- 
cgit v1.2.3


From 61b31f4e65f80d159831ace42b45812ba6beb995 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 1 Apr 2016 17:40:49 -0400
Subject: [pkg] update to versioneer 0.16

---
 common/MANIFEST.in                         |    1 +
 common/setup.cfg                           |    7 +
 common/setup.py                            |   63 +-
 common/src/leap/soledad/common/__init__.py |    4 +
 common/src/leap/soledad/common/_version.py |  549 ++++++--
 common/versioneer.py                       | 2059 +++++++++++++++++++++-------
 6 files changed, 2027 insertions(+), 656 deletions(-)

diff --git a/common/MANIFEST.in b/common/MANIFEST.in
index a26a12a6..93f46bbe 100644
--- a/common/MANIFEST.in
+++ b/common/MANIFEST.in
@@ -6,3 +6,4 @@ include CHANGELOG
 # What do we want the ddocs folder in the source package for? -- kali
 # it should be enough with having the compiled stuff.
 recursive-include src/leap/soledad/common/ddocs *
+include src/leap/soledad/common/_version.py
diff --git a/common/setup.cfg b/common/setup.cfg
index 3eb110b7..ed06f60d 100644
--- a/common/setup.cfg
+++ b/common/setup.cfg
@@ -8,3 +8,10 @@ ignore = E731
 [flake8]
 exclude = versioneer.py,_version.py,ddocs.py,*.egg,build,docs
 ignore = E731
+
+[versioneer]
+VCS = git
+style = pep440
+versionfile_source = src/leap/soledad/common/_version.py
+versionfile_build = leap/soledad/common/_version.py
+tag_prefix = 
diff --git a/common/setup.py b/common/setup.py
index 383745dc..8d9c4d6e 100644
--- a/common/setup.py
+++ b/common/setup.py
@@ -23,19 +23,15 @@ from os import listdir
 from os.path import realpath, dirname, isdir, join, isfile, basename
 import re
 
+from distutils.command.build import build as _build
 from setuptools import setup
 from setuptools import find_packages
 from setuptools import Command
 from setuptools.command.develop import develop as _cmd_develop
+import versioneer
 
 from pkg import utils
 
-import versioneer
-versioneer.versionfile_source = 'src/leap/soledad/common/_version.py'
-versioneer.versionfile_build = 'leap/soledad/common/_version.py'
-versioneer.tag_prefix = ''  # tags are like 1.2.0
-versioneer.parentdir_prefix = 'leap.soledad.common-'
-
 
 trove_classifiers = (
     "Development Status :: 3 - Alpha",
@@ -51,11 +47,11 @@ trove_classifiers = (
     "Topic :: Software Development :: Libraries :: Python Modules"
 )
 
-DOWNLOAD_BASE = ('https://github.com/leapcode/soledad/'
+DOWNLOAD_BASE = ('https://github.com/leapcode/bitmask_client/'
                  'archive/%s.tar.gz')
 _versions = versioneer.get_versions()
 VERSION = _versions['version']
-VERSION_FULL = _versions['full']
+VERSION_REVISION = _versions['full-revisionid']
 DOWNLOAD_URL = ""
 
 # get the short version for the download url
@@ -64,15 +60,30 @@ if len(_version_short) > 0:
     VERSION_SHORT = _version_short[0]
     DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT
 
-cmdclass = versioneer.get_cmdclass()
-
 
 class freeze_debianver(Command):
+
     """
     Freezes the version in a debian branch.
     To be used after merging the development branch onto the debian one.
     """
     user_options = []
+    template = r"""
+# This file was generated by the `freeze_debianver` command in setup.py
+# Using 'versioneer.py' (0.16) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+version_version = '{version}'
+full_revisionid = '{full_revisionid}'
+"""
+    templatefun = r"""
+
+def get_versions(default={}, verbose=False):
+        return {'version': version_version,
+                'full-revisionid': full_revisionid}
+"""
 
     def initialize_options(self):
         pass
@@ -86,28 +97,16 @@ class freeze_debianver(Command):
         if proceed != "y":
             print("He. You scared. Aborting.")
             return
-        template = r"""
-# This file was generated by the `freeze_debianver` command in setup.py
-# Using 'versioneer.py' (0.7+) from
-# revision-control system data, or from the parent directory name of an
-# unpacked source archive. Distribution tarballs contain a pre-generated copy
-# of this file.
-
-version_version = '{version}'
-version_full = '{version_full}'
-"""
-        templatefun = r"""
-
-def get_versions(default={}, verbose=False):
-        return {'version': version_version, 'full': version_full}
-"""
-        subst_template = template.format(
+        subst_template = self.template.format(
             version=VERSION_SHORT,
-            version_full=VERSION_FULL) + templatefun
-        with open(versioneer.versionfile_source, 'w') as f:
+            full_revisionid=VERSION_REVISION) + self.templatefun
+        versioneer_cfg = versioneer.get_config_from_root('.')
+        with open(versioneer_cfg.versionfile_source, 'w') as f:
             f.write(subst_template)
 
 
+cmdclass = versioneer.get_cmdclass()
+
 #
 # Couch backend design docs file generation.
 #
@@ -229,13 +228,9 @@ class cmd_develop(_cmd_develop):
         build_ddocs_py()
 
 
-# versioneer powered
-old_cmd_build = cmdclass["build"]
-
-
-class cmd_build(old_cmd_build):
+class cmd_build(_build):
     def run(self):
-        old_cmd_build.run(self)
+        _build.run(self)
         build_ddocs_py(basedir=self.build_lib, with_src=False)
 
 
diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py
index 1ba6ab89..d7f6929c 100644
--- a/common/src/leap/soledad/common/__init__.py
+++ b/common/src/leap/soledad/common/__init__.py
@@ -47,3 +47,7 @@ __all__ = [
     "soledad_assert_type",
     "__version__",
 ]
+
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions
diff --git a/common/src/leap/soledad/common/_version.py b/common/src/leap/soledad/common/_version.py
index a58af503..1168a88d 100644
--- a/common/src/leap/soledad/common/_version.py
+++ b/common/src/leap/soledad/common/_version.py
@@ -1,73 +1,157 @@
+
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
+
+"""Git implementation of _version.py."""
 
-# these strings will be replaced by git during git-archive
+import errno
+import os
+import re
 import subprocess
 import sys
-import re
-import os.path
 
-IN_LONG_VERSION_PY = True
 
-git_refnames = "$Format:%d$"
-git_full = "$Format:%H$"
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "$Format:%d$"
+    git_full = "$Format:%H$"
+    keywords = {"refnames": git_refnames, "full": git_full}
+    return keywords
 
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.cmd on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "pep440"
+    cfg.tag_prefix = ""
+    cfg.parentdir_prefix = "None"
+    cfg.versionfile_source = "src/leap/soledad/common/_version.py"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None
+    else:
         if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
+            print("unable to find command, tried %s" % (commands,))
         return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %s (error)" % args[0])
+            print("unable to run %s (error)" % dispcmd)
         return None
     return stdout
 
 
-def get_expanded_variables(versionfile_source):
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with "
+                  "prefix '%s'" % (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source, "r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {}  # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -83,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
         # "stabilization", as well as "HEAD" and "master".
         tags = set([r for r in refs if re.search(r'\d', r)])
         if verbose:
-            print("discarding '%s', no digits" % ",".join(refs - tags))
+            print("discarding '%s', no digits" % ",".join(refs-tags))
     if verbose:
         print("likely tags: %s" % ",".join(sorted(tags)))
     for ref in sorted(tags):
@@ -93,123 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             if verbose:
                 print("picking %s" % r)
             return {"version": r,
-                    "full": variables["full"].strip()}
-    # no suitable tags, so we use the full revision id
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return {"version": variables["full"].strip(),
-            "full": variables["full"].strip()}
-
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {}  # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(
-            os.path.join('..', here))
 
-    #
-    # XXX patch for our specific configuration with
-    # the three projects leap.soledad.{common, client, server}
-    # inside the same repo.
-    #
-    root = os.path.dirname(os.path.join('..', root))
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
 
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
     if not os.path.exists(os.path.join(root, ".git")):
         if verbose:
             print("no .git in %s" % root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.cmd"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" % (
-                stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source,
-                            verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {}  # without __file__, we have no hope
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%s*" % tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
+
+    try:
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
+
+    try:
+        root = os.path.realpath(__file__)
         # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
-    else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree"}
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with "
-                  "prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = ""
-parentdir_prefix = "leap.soledad.common-"
-versionfile_source = "src/leap/soledad/common/_version.py"
-
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = {"refnames": git_refnames, "full": git_full}
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version"}
diff --git a/common/versioneer.py b/common/versioneer.py
index 18dfd923..7ed2a21d 100644
--- a/common/versioneer.py
+++ b/common/versioneer.py
@@ -1,170 +1,641 @@
-#! /usr/bin/python
 
-"""versioneer.py
+# Version: 0.16
 
-(like a rocketeer, but for versions)
+"""The Versioneer - like a rocketeer, but for versions.
 
+The Versioneer
+==============
+
+* like a rocketeer, but for versions!
 * https://github.com/warner/python-versioneer
 * Brian Warner
 * License: Public Domain
-* Version: 0.7+
-
-This file helps distutils-based projects manage their version number by just
-creating version-control tags.
-
-For developers who work from a VCS-generated tree (e.g. 'git clone' etc),
-each 'setup.py version', 'setup.py build', 'setup.py sdist' will compute a
-version number by asking your version-control tool about the current
-checkout. The version number will be written into a generated _version.py
-file of your choosing, where it can be included by your __init__.py
-
-For users who work from a VCS-generated tarball (e.g. 'git archive'), it will
-compute a version number by looking at the name of the directory created when
-te tarball is unpacked. This conventionally includes both the name of the
-project and a version number.
-
-For users who work from a tarball built by 'setup.py sdist', it will get a
-version number from a previously-generated _version.py file.
-
-As a result, loading code directly from the source tree will not result in a
-real version. If you want real versions from VCS trees (where you frequently
-update from the upstream repository, or do new development), you will need to
-do a 'setup.py version' after each update, and load code from the build/
-directory.
-
-You need to provide this code with a few configuration values:
-
- versionfile_source:
-    A project-relative pathname into which the generated version strings
-    should be written. This is usually a _version.py next to your project's
-    main __init__.py file. If your project uses src/myproject/__init__.py,
-    this should be 'src/myproject/_version.py'. This file should be checked
-    in to your VCS as usual: the copy created below by 'setup.py
-    update_files' will include code that parses expanded VCS keywords in
-    generated tarballs. The 'build' and 'sdist' commands will replace it with
-    a copy that has just the calculated version string.
-
- versionfile_build:
-    Like versionfile_source, but relative to the build directory instead of
-    the source directory. These will differ when your setup.py uses
-    'package_dir='. If you have package_dir={'myproject': 'src/myproject'},
-    then you will probably have versionfile_build='myproject/_version.py' and
-    versionfile_source='src/myproject/_version.py'.
-
- tag_prefix: a string, like 'PROJECTNAME-', which appears at the start of all
-             VCS tags. If your tags look like 'myproject-1.2.0', then you
-             should use tag_prefix='myproject-'. If you use unprefixed tags
-             like '1.2.0', this should be an empty string.
-
- parentdir_prefix: a string, frequently the same as tag_prefix, which
-                   appears at the start of all unpacked tarball filenames. If
-                   your tarball unpacks into 'myproject-1.2.0', this should
-                   be 'myproject-'.
-
-To use it:
-
- 1: include this file in the top level of your project
- 2: make the following changes to the top of your setup.py:
-     import versioneer
-     versioneer.versionfile_source = 'src/myproject/_version.py'
-     versioneer.versionfile_build = 'myproject/_version.py'
-     versioneer.tag_prefix = '' # tags are like 1.2.0
-     versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0'
- 3: add the following arguments to the setup() call in your setup.py:
-     version=versioneer.get_version(),
-     cmdclass=versioneer.get_cmdclass(),
- 4: run 'setup.py update_files', which will create _version.py, and will
-    modify your __init__.py to define __version__ (by calling a function
-    from _version.py)
- 5: modify your MANIFEST.in to include versioneer.py
- 6: add both versioneer.py and the generated _version.py to your VCS
-"""
+* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy
+* [![Latest Version]
+(https://pypip.in/version/versioneer/badge.svg?style=flat)
+](https://pypi.python.org/pypi/versioneer/)
+* [![Build Status]
+(https://travis-ci.org/warner/python-versioneer.png?branch=master)
+](https://travis-ci.org/warner/python-versioneer)
+
+This is a tool for managing a recorded version number in distutils-based
+python projects. The goal is to remove the tedious and error-prone "update
+the embedded version string" step from your release process. Making a new
+release should be as easy as recording a new tag in your version-control
+system, and maybe making new tarballs.
+
+
+## Quick Install
+
+* `pip install versioneer` to somewhere to your $PATH
+* add a `[versioneer]` section to your setup.cfg (see below)
+* run `versioneer install` in your source tree, commit the results
+
+## Version Identifiers
+
+Source trees come from a variety of places:
+
+* a version-control system checkout (mostly used by developers)
+* a nightly tarball, produced by build automation
+* a snapshot tarball, produced by a web-based VCS browser, like github's
+  "tarball from tag" feature
+* a release tarball, produced by "setup.py sdist", distributed through PyPI
+
+Within each source tree, the version identifier (either a string or a number,
+this tool is format-agnostic) can come from a variety of places:
+
+* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows
+  about recent "tags" and an absolute revision-id
+* the name of the directory into which the tarball was unpacked
+* an expanded VCS keyword ($Id$, etc)
+* a `_version.py` created by some earlier build step
+
+For released software, the version identifier is closely related to a VCS
+tag. Some projects use tag names that include more than just the version
+string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool
+needs to strip the tag prefix to extract the version identifier. For
+unreleased software (between tags), the version identifier should provide
+enough information to help developers recreate the same tree, while also
+giving them an idea of roughly how old the tree is (after version 1.2, before
+version 1.3). Many VCS systems can report a description that captures this,
+for example `git describe --tags --dirty --always` reports things like
+"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the
+0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has
+uncommitted changes.
+
+The version identifier is used for multiple purposes:
+
+* to allow the module to self-identify its version: `myproject.__version__`
+* to choose a name and prefix for a 'setup.py sdist' tarball
+
+## Theory of Operation
+
+Versioneer works by adding a special `_version.py` file into your source
+tree, where your `__init__.py` can import it. This `_version.py` knows how to
+dynamically ask the VCS tool for version information at import time.
+
+`_version.py` also contains `$Revision$` markers, and the installation
+process marks `_version.py` to have this marker rewritten with a tag name
+during the `git archive` command. As a result, generated tarballs will
+contain enough information to get the proper version.
+
+To allow `setup.py` to compute a version too, a `versioneer.py` is added to
+the top level of your source tree, next to `setup.py` and the `setup.cfg`
+that configures it. This overrides several distutils/setuptools commands to
+compute the version when invoked, and changes `setup.py build` and `setup.py
+sdist` to replace `_version.py` with a small static file that contains just
+the generated version data.
+
+## Installation
+
+First, decide on values for the following configuration variables:
+
+* `VCS`: the version control system you use. Currently accepts "git".
+
+* `style`: the style of version string to be produced. See "Styles" below for
+  details. Defaults to "pep440", which looks like
+  `TAG[+DISTANCE.gSHORTHASH[.dirty]]`.
+
+* `versionfile_source`:
+
+  A project-relative pathname into which the generated version strings should
+  be written. This is usually a `_version.py` next to your project's main
+  `__init__.py` file, so it can be imported at runtime. If your project uses
+  `src/myproject/__init__.py`, this should be `src/myproject/_version.py`.
+  This file should be checked in to your VCS as usual: the copy created below
+  by `setup.py setup_versioneer` will include code that parses expanded VCS
+  keywords in generated tarballs. The 'build' and 'sdist' commands will
+  replace it with a copy that has just the calculated version string.
+
+  This must be set even if your project does not have any modules (and will
+  therefore never import `_version.py`), since "setup.py sdist" -based trees
+  still need somewhere to record the pre-calculated version strings. Anywhere
+  in the source tree should do. If there is a `__init__.py` next to your
+  `_version.py`, the `setup.py setup_versioneer` command (described below)
+  will append some `__version__`-setting assignments, if they aren't already
+  present.
+
+* `versionfile_build`:
+
+  Like `versionfile_source`, but relative to the build directory instead of
+  the source directory. These will differ when your setup.py uses
+  'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`,
+  then you will probably have `versionfile_build='myproject/_version.py'` and
+  `versionfile_source='src/myproject/_version.py'`.
+
+  If this is set to None, then `setup.py build` will not attempt to rewrite
+  any `_version.py` in the built tree. If your project does not have any
+  libraries (e.g. if it only builds a script), then you should use
+  `versionfile_build = None`. To actually use the computed version string,
+  your `setup.py` will need to override `distutils.command.build_scripts`
+  with a subclass that explicitly inserts a copy of
+  `versioneer.get_version()` into your script file. See
+  `test/demoapp-script-only/setup.py` for an example.
+
+* `tag_prefix`:
+
+  a string, like 'PROJECTNAME-', which appears at the start of all VCS tags.
+  If your tags look like 'myproject-1.2.0', then you should use
+  tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this
+  should be an empty string, using either `tag_prefix=` or `tag_prefix=''`.
+
+* `parentdir_prefix`:
+
+  a optional string, frequently the same as tag_prefix, which appears at the
+  start of all unpacked tarball filenames. If your tarball unpacks into
+  'myproject-1.2.0', this should be 'myproject-'. To disable this feature,
+  just omit the field from your `setup.cfg`.
+
+This tool provides one script, named `versioneer`. That script has one mode,
+"install", which writes a copy of `versioneer.py` into the current directory
+and runs `versioneer.py setup` to finish the installation.
+
+To versioneer-enable your project:
+
+* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and
+  populating it with the configuration values you decided earlier (note that
+  the option names are not case-sensitive):
+
+  ````
+  [versioneer]
+  VCS = git
+  style = pep440
+  versionfile_source = src/myproject/_version.py
+  versionfile_build = myproject/_version.py
+  tag_prefix =
+  parentdir_prefix = myproject-
+  ````
+
+* 2: Run `versioneer install`. This will do the following:
+
+  * copy `versioneer.py` into the top of your source tree
+  * create `_version.py` in the right place (`versionfile_source`)
+  * modify your `__init__.py` (if one exists next to `_version.py`) to define
+    `__version__` (by calling a function from `_version.py`)
+  * modify your `MANIFEST.in` to include both `versioneer.py` and the
+    generated `_version.py` in sdist tarballs
+
+  `versioneer install` will complain about any problems it finds with your
+  `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all
+  the problems.
+
+* 3: add a `import versioneer` to your setup.py, and add the following
+  arguments to the setup() call:
+
+        version=versioneer.get_version(),
+        cmdclass=versioneer.get_cmdclass(),
+
+* 4: commit these changes to your VCS. To make sure you won't forget,
+  `versioneer install` will mark everything it touched for addition using
+  `git add`. Don't forget to add `setup.py` and `setup.cfg` too.
+
+## Post-Installation Usage
+
+Once established, all uses of your tree from a VCS checkout should get the
+current version string. All generated tarballs should include an embedded
+version string (so users who unpack them will not need a VCS tool installed).
+
+If you distribute your project through PyPI, then the release process should
+boil down to two steps:
+
+* 1: git tag 1.0
+* 2: python setup.py register sdist upload
+
+If you distribute it through github (i.e. users use github to generate
+tarballs with `git archive`), the process is:
+
+* 1: git tag 1.0
+* 2: git push; git push --tags
+
+Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at
+least one tag in its history.
+
+## Version-String Flavors
+
+Code which uses Versioneer can learn about its version string at runtime by
+importing `_version` from your main `__init__.py` file and running the
+`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can
+import the top-level `versioneer.py` and run `get_versions()`.
+
+Both functions return a dictionary with different flavors of version
+information:
+
+* `['version']`: A condensed version string, rendered using the selected
+  style. This is the most commonly used value for the project's version
+  string. The default "pep440" style yields strings like `0.11`,
+  `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section
+  below for alternative styles.
+
+* `['full-revisionid']`: detailed revision identifier. For Git, this is the
+  full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac".
+
+* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that
+  this is only accurate if run in a VCS checkout, otherwise it is likely to
+  be False or None
+
+* `['error']`: if the version string could not be computed, this will be set
+  to a string describing the problem, otherwise it will be None. It may be
+  useful to throw an exception in setup.py if this is set, to avoid e.g.
+  creating tarballs with a version string of "unknown".
+
+Some variants are more useful than others. Including `full-revisionid` in a
+bug report should allow developers to reconstruct the exact code being tested
+(or indicate the presence of local changes that should be shared with the
+developers). `version` is suitable for display in an "about" box or a CLI
+`--version` output: it can be easily compared against release notes and lists
+of bugs fixed in various releases.
+
+The installer adds the following text to your `__init__.py` to place a basic
+version in `YOURPROJECT.__version__`:
+
+    from ._version import get_versions
+    __version__ = get_versions()['version']
+    del get_versions
+
+## Styles
+
+The setup.cfg `style=` configuration controls how the VCS information is
+rendered into a version string.
+
+The default style, "pep440", produces a PEP440-compliant string, equal to the
+un-prefixed tag name for actual releases, and containing an additional "local
+version" section with more detail for in-between builds. For Git, this is
+TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags
+--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the
+tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and
+that this commit is two revisions ("+2") beyond the "0.11" tag. For released
+software (exactly equal to a known tag), the identifier will only contain the
+stripped tag, e.g. "0.11".
+
+Other styles are available. See details.md in the Versioneer source tree for
+descriptions.
+
+## Debugging
+
+Versioneer tries to avoid fatal errors: if something goes wrong, it will tend
+to return a version of "0+unknown". To investigate the problem, run `setup.py
+version`, which will run the version-lookup code in a verbose mode, and will
+display the full contents of `get_versions()` (including the `error` string,
+which may help identify what went wrong).
+
+## Updating Versioneer
+
+To upgrade your project to a new release of Versioneer, do the following:
+
+* install the new Versioneer (`pip install -U versioneer` or equivalent)
+* edit `setup.cfg`, if necessary, to include any new configuration settings
+  indicated by the release notes
+* re-run `versioneer install` in your source tree, to replace
+  `SRC/_version.py`
+* commit any changed files
+
+### Upgrading to 0.16
+
+Nothing special.
+
+### Upgrading to 0.15
+
+Starting with this version, Versioneer is configured with a `[versioneer]`
+section in your `setup.cfg` file. Earlier versions required the `setup.py` to
+set attributes on the `versioneer` module immediately after import. The new
+version will refuse to run (raising an exception during import) until you
+have provided the necessary `setup.cfg` section.
+
+In addition, the Versioneer package provides an executable named
+`versioneer`, and the installation process is driven by running `versioneer
+install`. In 0.14 and earlier, the executable was named
+`versioneer-installer` and was run without an argument.
+
+### Upgrading to 0.14
 
-import os, sys, re
-from distutils.core import Command
-from distutils.command.sdist import sdist as _sdist
-from distutils.command.build import build as _build
+0.14 changes the format of the version string. 0.13 and earlier used
+hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a
+plus-separated "local version" section strings, with dot-separated
+components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old
+format, but should be ok with the new one.
 
-versionfile_source = None
-versionfile_build = None
-tag_prefix = None
-parentdir_prefix = None
+### Upgrading from 0.11 to 0.12
 
-VCS = "git"
-IN_LONG_VERSION_PY = False
+Nothing special.
 
+### Upgrading from 0.10 to 0.11
 
-LONG_VERSION_PY = '''
-IN_LONG_VERSION_PY = True
+You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running
+`setup.py setup_versioneer`. This will enable the use of additional
+version-control systems (SVN, etc) in the future.
+
+## Future Directions
+
+This tool is designed to make it easily extended to other version-control
+systems: all VCS-specific components are in separate directories like
+src/git/ . The top-level `versioneer.py` script is assembled from these
+components by running make-versioneer.py . In the future, make-versioneer.py
+will take a VCS name as an argument, and will construct a version of
+`versioneer.py` that is specific to the given VCS. It might also take the
+configuration arguments that are currently provided manually during
+installation by editing setup.py . Alternatively, it might go the other
+direction and include code from all supported VCS systems, reducing the
+number of intermediate scripts.
+
+
+## License
+
+To make Versioneer easier to embed, all its code is dedicated to the public
+domain. The `_version.py` that it creates is also in the public domain.
+Specifically, both are released under the Creative Commons "Public Domain
+Dedication" license (CC0-1.0), as described in
+https://creativecommons.org/publicdomain/zero/1.0/ .
+
+"""
+
+from __future__ import print_function
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+import errno
+import json
+import os
+import re
+import subprocess
+import sys
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_root():
+    """Get the project root directory.
+
+    We require that all commands are run from the project root, i.e. the
+    directory that contains setup.py, setup.cfg, and versioneer.py .
+    """
+    root = os.path.realpath(os.path.abspath(os.getcwd()))
+    setup_py = os.path.join(root, "setup.py")
+    versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        # allow 'python path/to/setup.py COMMAND'
+        root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
+        setup_py = os.path.join(root, "setup.py")
+        versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        err = ("Versioneer was unable to run the project root directory. "
+               "Versioneer requires setup.py to be executed from "
+               "its immediate directory (like 'python setup.py COMMAND'), "
+               "or in a way that lets it use sys.argv[0] to find the root "
+               "(like 'python path/to/setup.py COMMAND').")
+        raise VersioneerBadRootError(err)
+    try:
+        # Certain runtime workflows (setup.py install/develop in a setuptools
+        # tree) execute all dependencies in a single python process, so
+        # "versioneer" may be imported multiple times, and python's shared
+        # module-import table will cache the first one. So we can't use
+        # os.path.dirname(__file__), as that will find whichever
+        # versioneer.py was first imported, even in later projects.
+        me = os.path.realpath(os.path.abspath(__file__))
+        if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]:
+            print("Warning: build in %s is using versioneer.py from %s"
+                  % (os.path.dirname(me), versioneer_py))
+    except NameError:
+        pass
+    return root
+
+
+def get_config_from_root(root):
+    """Read the project setup.cfg file to determine Versioneer config."""
+    # This might raise EnvironmentError (if setup.cfg is missing), or
+    # configparser.NoSectionError (if it lacks a [versioneer] section), or
+    # configparser.NoOptionError (if it lacks "VCS="). See the docstring at
+    # the top of versioneer.py for instructions on writing your setup.cfg .
+    setup_cfg = os.path.join(root, "setup.cfg")
+    parser = configparser.SafeConfigParser()
+    with open(setup_cfg, "r") as f:
+        parser.readfp(f)
+    VCS = parser.get("versioneer", "VCS")  # mandatory
+
+    def get(parser, name):
+        if parser.has_option("versioneer", name):
+            return parser.get("versioneer", name)
+        return None
+    cfg = VersioneerConfig()
+    cfg.VCS = VCS
+    cfg.style = get(parser, "style") or ""
+    cfg.versionfile_source = get(parser, "versionfile_source")
+    cfg.versionfile_build = get(parser, "versionfile_build")
+    cfg.tag_prefix = get(parser, "tag_prefix")
+    if cfg.tag_prefix in ("''", '""'):
+        cfg.tag_prefix = ""
+    cfg.parentdir_prefix = get(parser, "parentdir_prefix")
+    cfg.verbose = get(parser, "verbose")
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+# these dictionaries contain VCS-specific tools
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None
+    else:
+        if verbose:
+            print("unable to find command, tried %s" % (commands,))
+        return None
+    stdout = p.communicate()[0].strip()
+    if sys.version_info[0] >= 3:
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %s (error)" % dispcmd)
+        return None
+    return stdout
+LONG_VERSION_PY['git'] = '''
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
-
-# these strings will be replaced by git during git-archive
-git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
-git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
 
+"""Git implementation of _version.py."""
 
+import errno
+import os
+import re
 import subprocess
 import sys
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.exe on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
+    git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+    keywords = {"refnames": git_refnames, "full": git_full}
+    return keywords
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "%(STYLE)s"
+    cfg.tag_prefix = "%(TAG_PREFIX)s"
+    cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s"
+    cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %%s" %% dispcmd)
+                print(e)
+            return None
+    else:
         if verbose:
-            print("unable to run %%s" %% args[0])
-            print(e)
+            print("unable to find command, tried %%s" %% (commands,))
         return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %%s (error)" %% args[0])
+            print("unable to run %%s (error)" %% dispcmd)
         return None
     return stdout
 
 
-import sys
-import re
-import os.path
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%%s', but '%%s' doesn't start with "
+                  "prefix '%%s'" %% (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
+
 
-def get_expanded_variables(versionfile_source):
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source,"r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -189,172 +660,350 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %%s" %% r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
     if not os.path.exists(os.path.join(root, ".git")):
         if verbose:
             print("no .git in %%s" %% root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
-        # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%%s*" %% tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%%s'"
+                               %% describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%%s' doesn't start with prefix '%%s'"
+                print(fmt %% (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'"
+                               %% (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
     else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+        # exception #1
+        rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%%d" %% pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%%d" %% pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%%s" %% pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%%s" %% pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %%
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = "%(TAG_PREFIX)s"
-parentdir_prefix = "%(PARENTDIR_PREFIX)s"
-versionfile_source = "%(VERSIONFILE_SOURCE)s"
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = { "refnames": git_refnames, "full": git_full }
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
 
-'''
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
 
+    Like 'git describe --tags --dirty --always'.
 
-import subprocess
-import sys
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%%s'" %% style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
 
-def run_command(args, cwd=None, verbose=False):
     try:
-        # remember shell=False, so use git.exe on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
-        if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
-        return None
-    stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
-        stdout = stdout.decode()
-    if p.returncode != 0:
-        if verbose:
-            print("unable to run %s (error)" % args[0])
-        return None
-    return stdout
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
 
+    try:
+        root = os.path.realpath(__file__)
+        # versionfile_source is the relative path from the top of the source
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
+            root = os.path.dirname(root)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree"}
 
-import sys
-import re
-import os.path
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
 
-def get_expanded_variables(versionfile_source):
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version"}
+'''
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source,"r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -379,117 +1028,122 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %s" % r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = os.path.dirname(
-        os.path.join('..', here))
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(
-            os.path.join('..', here))
 
-    ######################################################
-    # XXX patch for our specific configuration with
-    # the three projects leap.soledad.{common, client, server}
-    # inside the same repo.
-    ######################################################
-    root = os.path.dirname(os.path.join('..', root))
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
 
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
     if not os.path.exists(os.path.join(root, ".git")):
         if verbose:
             print("no .git in %s" % root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(os.path.join('..', __file__))
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
-        # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%s*" % tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
     else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
+    return pieces
 
-import sys
 
-def do_vcs_install(versionfile_source, ipy):
-    GIT = "git"
+def do_vcs_install(manifest_in, versionfile_source, ipy):
+    """Git-specific installation logic for Versioneer.
+
+    For Git, this means creating/changing .gitattributes to mark _version.py
+    for export-time keyword substitution.
+    """
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    run_command([GIT, "add", "versioneer.py"])
-    run_command([GIT, "add", versionfile_source])
-    run_command([GIT, "add", ipy])
+        GITS = ["git.cmd", "git.exe"]
+    files = [manifest_in, versionfile_source]
+    if ipy:
+        files.append(ipy)
+    try:
+        me = __file__
+        if me.endswith(".pyc") or me.endswith(".pyo"):
+            me = os.path.splitext(me)[0] + ".py"
+        versioneer_file = os.path.relpath(me)
+    except NameError:
+        versioneer_file = "versioneer.py"
+    files.append(versioneer_file)
     present = False
     try:
         f = open(".gitattributes", "r")
@@ -504,135 +1158,487 @@ def do_vcs_install(versionfile_source, ipy):
         f = open(".gitattributes", "a+")
         f.write("%s export-subst\n" % versionfile_source)
         f.close()
-        run_command([GIT, "add", ".gitattributes"])
+        files.append(".gitattributes")
+    run_command(GITS, ["add", "--"] + files)
+
 
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with "
+                  "prefix '%s'" % (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
 
 SHORT_VERSION_PY = """
-# This file was generated by 'versioneer.py' (0.7+) from
+# This file was generated by 'versioneer.py' (0.16) from
 # revision-control system data, or from the parent directory name of an
 # unpacked source archive. Distribution tarballs contain a pre-generated copy
 # of this file.
 
-version_version = '%(version)s'
-version_full = '%(full)s'
-def get_versions(default={}, verbose=False):
-    return {'version': version_version, 'full': version_full}
+import json
+import sys
+
+version_json = '''
+%s
+'''  # END VERSION_JSON
 
+
+def get_versions():
+    return json.loads(version_json)
 """
 
-DEFAULT = {"version": "unknown", "full": "unknown"}
 
 def versions_from_file(filename):
-    versions = {}
+    """Try to determine the version from _version.py if present."""
     try:
-        f = open(filename)
+        with open(filename) as f:
+            contents = f.read()
     except EnvironmentError:
-        return versions
-    for line in f.readlines():
-        mo = re.match("version_version = '([^']+)'", line)
-        if mo:
-            versions["version"] = mo.group(1)
-        mo = re.match("version_full = '([^']+)'", line)
-        if mo:
-            versions["full"] = mo.group(1)
-    f.close()
-    return versions
+        raise NotThisMethod("unable to read _version.py")
+    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
+                   contents, re.M | re.S)
+    if not mo:
+        raise NotThisMethod("no version_json in _version.py")
+    return json.loads(mo.group(1))
+
 
 def write_to_version_file(filename, versions):
-    f = open(filename, "w")
-    f.write(SHORT_VERSION_PY % versions)
-    f.close()
+    """Write the given version number to the given _version.py file."""
+    os.unlink(filename)
+    contents = json.dumps(versions, sort_keys=True,
+                          indent=1, separators=(",", ": "))
+    with open(filename, "w") as f:
+        f.write(SHORT_VERSION_PY % contents)
+
     print("set %s to '%s'" % (filename, versions["version"]))
 
 
-def get_best_versions(versionfile, tag_prefix, parentdir_prefix,
-                      default=DEFAULT, verbose=False):
-    # returns dict with two keys: 'version' and 'full'
-    #
-    # extract version from first of _version.py, 'git describe', parentdir.
-    # This is meant to work for developers using a source checkout, for users
-    # of a tarball created by 'setup.py sdist', and for users of a
-    # tarball/zipball created by 'git archive' or github's download-from-tag
-    # feature.
-
-    variables = get_expanded_variables(versionfile_source)
-    if variables:
-        ver = versions_from_expanded_variables(variables, tag_prefix)
-        if ver:
-            if verbose: print("got version from expanded variable %s" % ver)
-            return ver
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
 
-    ver = versions_from_file(versionfile)
-    if ver:
-        if verbose: print("got version from file %s %s" % (versionfile, ver))
-        return ver
 
-    ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if ver:
-        if verbose: print("got version from git %s" % ver)
-        return ver
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
 
-    ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose)
-    if ver:
-        if verbose: print("got version from parentdir %s" % ver)
-        return ver
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
 
-    if verbose: print("got version from default %s" % ver)
-    return default
-
-def get_versions(default=DEFAULT, verbose=False):
-    assert versionfile_source is not None, "please set versioneer.versionfile_source"
-    assert tag_prefix is not None, "please set versioneer.tag_prefix"
-    assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix"
-    return get_best_versions(versionfile_source, tag_prefix, parentdir_prefix,
-                             default=default, verbose=verbose)
-def get_version(verbose=False):
-    return get_versions(verbose=verbose)["version"]
-
-class cmd_version(Command):
-    description = "report generated version string"
-    user_options = []
-    boolean_options = []
-    def initialize_options(self):
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+class VersioneerBadRootError(Exception):
+    """The project root directory is unknown or missing key files."""
+
+
+def get_versions(verbose=False):
+    """Get the project version from whatever source is available.
+
+    Returns dict with two keys: 'version' and 'full'.
+    """
+    if "versioneer" in sys.modules:
+        # see the discussion in cmdclass.py:get_cmdclass()
+        del sys.modules["versioneer"]
+
+    root = get_root()
+    cfg = get_config_from_root(root)
+
+    assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg"
+    handlers = HANDLERS.get(cfg.VCS)
+    assert handlers, "unrecognized VCS '%s'" % cfg.VCS
+    verbose = verbose or cfg.verbose
+    assert cfg.versionfile_source is not None, \
+        "please set versioneer.versionfile_source"
+    assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix"
+
+    versionfile_abs = os.path.join(root, cfg.versionfile_source)
+
+    # extract version from first of: _version.py, VCS command (e.g. 'git
+    # describe'), parentdir. This is meant to work for developers using a
+    # source checkout, for users of a tarball created by 'setup.py sdist',
+    # and for users of a tarball/zipball created by 'git archive' or github's
+    # download-from-tag feature or the equivalent in other VCSes.
+
+    get_keywords_f = handlers.get("get_keywords")
+    from_keywords_f = handlers.get("keywords")
+    if get_keywords_f and from_keywords_f:
+        try:
+            keywords = get_keywords_f(versionfile_abs)
+            ver = from_keywords_f(keywords, cfg.tag_prefix, verbose)
+            if verbose:
+                print("got version from expanded keyword %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        ver = versions_from_file(versionfile_abs)
+        if verbose:
+            print("got version from file %s %s" % (versionfile_abs, ver))
+        return ver
+    except NotThisMethod:
         pass
-    def finalize_options(self):
+
+    from_vcs_f = handlers.get("pieces_from_vcs")
+    if from_vcs_f:
+        try:
+            pieces = from_vcs_f(cfg.tag_prefix, root, verbose)
+            ver = render(pieces, cfg.style)
+            if verbose:
+                print("got version from VCS %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        if cfg.parentdir_prefix:
+            ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+            if verbose:
+                print("got version from parentdir %s" % ver)
+            return ver
+    except NotThisMethod:
         pass
-    def run(self):
-        ver = get_version(verbose=True)
-        print("Version is currently: %s" % ver)
-
-
-class cmd_build(_build):
-    def run(self):
-        versions = get_versions(verbose=True)
-        _build.run(self)
-        # now locate _version.py in the new build/ directory and replace it
-        # with an updated value
-        target_versionfile = os.path.join(self.build_lib, versionfile_build)
-        print("UPDATING %s" % target_versionfile)
-        os.unlink(target_versionfile)
-        f = open(target_versionfile, "w")
-        f.write(SHORT_VERSION_PY % versions)
-        f.close()
 
-class cmd_sdist(_sdist):
-    def run(self):
-        versions = get_versions(verbose=True)
-        self._versioneer_generated_versions = versions
-        # unless we update this, the command will keep using the old version
-        self.distribution.metadata.version = versions["version"]
-        return _sdist.run(self)
-
-    def make_release_tree(self, base_dir, files):
-        _sdist.make_release_tree(self, base_dir, files)
-        # now locate _version.py in the new base_dir directory (remembering
-        # that it may be a hardlink) and replace it with an updated value
-        target_versionfile = os.path.join(base_dir, versionfile_source)
-        print("UPDATING %s" % target_versionfile)
-        os.unlink(target_versionfile)
-        f = open(target_versionfile, "w")
-        f.write(SHORT_VERSION_PY % self._versioneer_generated_versions)
-        f.close()
+    if verbose:
+        print("unable to compute version")
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None, "error": "unable to compute version"}
+
+
+def get_version():
+    """Get the short version string for this project."""
+    return get_versions()["version"]
+
+
+def get_cmdclass():
+    """Get the custom setuptools/distutils subclasses used by Versioneer."""
+    if "versioneer" in sys.modules:
+        del sys.modules["versioneer"]
+        # this fixes the "python setup.py develop" case (also 'install' and
+        # 'easy_install .'), in which subdependencies of the main project are
+        # built (using setup.py bdist_egg) in the same python process. Assume
+        # a main project A and a dependency B, which use different versions
+        # of Versioneer. A's setup.py imports A's Versioneer, leaving it in
+        # sys.modules by the time B's setup.py is executed, causing B to run
+        # with the wrong versioneer. Setuptools wraps the sub-dep builds in a
+        # sandbox that restores sys.modules to it's pre-build state, so the
+        # parent is protected against the child's "import versioneer". By
+        # removing ourselves from sys.modules here, before the child build
+        # happens, we protect the child from the parent's versioneer too.
+        # Also see https://github.com/warner/python-versioneer/issues/52
+
+    cmds = {}
+
+    # we add "version" to both distutils and setuptools
+    from distutils.core import Command
+
+    class cmd_version(Command):
+        description = "report generated version string"
+        user_options = []
+        boolean_options = []
+
+        def initialize_options(self):
+            pass
+
+        def finalize_options(self):
+            pass
+
+        def run(self):
+            vers = get_versions(verbose=True)
+            print("Version: %s" % vers["version"])
+            print(" full-revisionid: %s" % vers.get("full-revisionid"))
+            print(" dirty: %s" % vers.get("dirty"))
+            if vers["error"]:
+                print(" error: %s" % vers["error"])
+    cmds["version"] = cmd_version
+
+    # we override "build_py" in both distutils and setuptools
+    #
+    # most invocation pathways end up running build_py:
+    #  distutils/build -> build_py
+    #  distutils/install -> distutils/build ->..
+    #  setuptools/bdist_wheel -> distutils/install ->..
+    #  setuptools/bdist_egg -> distutils/install_lib -> build_py
+    #  setuptools/install -> bdist_egg ->..
+    #  setuptools/develop -> ?
+
+    # we override different "build_py" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.build_py import build_py as _build_py
+    else:
+        from distutils.command.build_py import build_py as _build_py
+
+    class cmd_build_py(_build_py):
+        def run(self):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            versions = get_versions()
+            _build_py.run(self)
+            # now locate _version.py in the new build/ directory and replace
+            # it with an updated value
+            if cfg.versionfile_build:
+                target_versionfile = os.path.join(self.build_lib,
+                                                  cfg.versionfile_build)
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+    cmds["build_py"] = cmd_build_py
+
+    if "cx_Freeze" in sys.modules:  # cx_freeze enabled?
+        from cx_Freeze.dist import build_exe as _build_exe
+
+        class cmd_build_exe(_build_exe):
+            def run(self):
+                root = get_root()
+                cfg = get_config_from_root(root)
+                versions = get_versions()
+                target_versionfile = cfg.versionfile_source
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+
+                _build_exe.run(self)
+                os.unlink(target_versionfile)
+                with open(cfg.versionfile_source, "w") as f:
+                    LONG = LONG_VERSION_PY[cfg.VCS]
+                    f.write(LONG %
+                            {"DOLLAR": "$",
+                             "STYLE": cfg.style,
+                             "TAG_PREFIX": cfg.tag_prefix,
+                             "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                             "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                             })
+        cmds["build_exe"] = cmd_build_exe
+        del cmds["build_py"]
+
+    # we override different "sdist" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.sdist import sdist as _sdist
+    else:
+        from distutils.command.sdist import sdist as _sdist
+
+    class cmd_sdist(_sdist):
+        def run(self):
+            versions = get_versions()
+            self._versioneer_generated_versions = versions
+            # unless we update this, the command will keep using the old
+            # version
+            self.distribution.metadata.version = versions["version"]
+            return _sdist.run(self)
+
+        def make_release_tree(self, base_dir, files):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            _sdist.make_release_tree(self, base_dir, files)
+            # now locate _version.py in the new base_dir directory
+            # (remembering that it may be a hardlink) and replace it with an
+            # updated value
+            target_versionfile = os.path.join(base_dir, cfg.versionfile_source)
+            print("UPDATING %s" % target_versionfile)
+            write_to_version_file(target_versionfile,
+                                  self._versioneer_generated_versions)
+    cmds["sdist"] = cmd_sdist
+
+    return cmds
+
+
+CONFIG_ERROR = """
+setup.cfg is missing the necessary Versioneer configuration. You need
+a section like:
+
+ [versioneer]
+ VCS = git
+ style = pep440
+ versionfile_source = src/myproject/_version.py
+ versionfile_build = myproject/_version.py
+ tag_prefix =
+ parentdir_prefix = myproject-
+
+You will also need to edit your setup.py to use the results:
+
+ import versioneer
+ setup(version=versioneer.get_version(),
+       cmdclass=versioneer.get_cmdclass(), ...)
+
+Please read the docstring in ./versioneer.py for configuration instructions,
+edit setup.cfg, and re-run the installer or 'python versioneer.py setup'.
+"""
+
+SAMPLE_CONFIG = """
+# See the docstring in versioneer.py for instructions. Note that you must
+# re-run 'versioneer.py setup' after changing this section, and commit the
+# resulting files.
+
+[versioneer]
+#VCS = git
+#style = pep440
+#versionfile_source =
+#versionfile_build =
+#tag_prefix =
+#parentdir_prefix =
+
+"""
 
 INIT_PY_SNIPPET = """
 from ._version import get_versions
@@ -640,40 +1646,129 @@ __version__ = get_versions()['version']
 del get_versions
 """
 
-class cmd_update_files(Command):
-    description = "modify __init__.py and create _version.py"
-    user_options = []
-    boolean_options = []
-    def initialize_options(self):
-        pass
-    def finalize_options(self):
-        pass
-    def run(self):
-        ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py")
-        print(" creating %s" % versionfile_source)
-        f = open(versionfile_source, "w")
-        f.write(LONG_VERSION_PY % {"DOLLAR": "$",
-                                   "TAG_PREFIX": tag_prefix,
-                                   "PARENTDIR_PREFIX": parentdir_prefix,
-                                   "VERSIONFILE_SOURCE": versionfile_source,
-                                   })
-        f.close()
+
+def do_setup():
+    """Main VCS-independent setup function for installing Versioneer."""
+    root = get_root()
+    try:
+        cfg = get_config_from_root(root)
+    except (EnvironmentError, configparser.NoSectionError,
+            configparser.NoOptionError) as e:
+        if isinstance(e, (EnvironmentError, configparser.NoSectionError)):
+            print("Adding sample versioneer config to setup.cfg",
+                  file=sys.stderr)
+            with open(os.path.join(root, "setup.cfg"), "a") as f:
+                f.write(SAMPLE_CONFIG)
+        print(CONFIG_ERROR, file=sys.stderr)
+        return 1
+
+    print(" creating %s" % cfg.versionfile_source)
+    with open(cfg.versionfile_source, "w") as f:
+        LONG = LONG_VERSION_PY[cfg.VCS]
+        f.write(LONG % {"DOLLAR": "$",
+                        "STYLE": cfg.style,
+                        "TAG_PREFIX": cfg.tag_prefix,
+                        "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                        "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                        })
+
+    ipy = os.path.join(os.path.dirname(cfg.versionfile_source),
+                       "__init__.py")
+    if os.path.exists(ipy):
         try:
-            old = open(ipy, "r").read()
+            with open(ipy, "r") as f:
+                old = f.read()
         except EnvironmentError:
             old = ""
         if INIT_PY_SNIPPET not in old:
             print(" appending to %s" % ipy)
-            f = open(ipy, "a")
-            f.write(INIT_PY_SNIPPET)
-            f.close()
+            with open(ipy, "a") as f:
+                f.write(INIT_PY_SNIPPET)
         else:
             print(" %s unmodified" % ipy)
-        do_vcs_install(versionfile_source, ipy)
+    else:
+        print(" %s doesn't exist, ok" % ipy)
+        ipy = None
+
+    # Make sure both the top-level "versioneer.py" and versionfile_source
+    # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so
+    # they'll be copied into source distributions. Pip won't be able to
+    # install the package without this.
+    manifest_in = os.path.join(root, "MANIFEST.in")
+    simple_includes = set()
+    try:
+        with open(manifest_in, "r") as f:
+            for line in f:
+                if line.startswith("include "):
+                    for include in line.split()[1:]:
+                        simple_includes.add(include)
+    except EnvironmentError:
+        pass
+    # That doesn't cover everything MANIFEST.in can do
+    # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
+    # it might give some false negatives. Appending redundant 'include'
+    # lines is safe, though.
+    if "versioneer.py" not in simple_includes:
+        print(" appending 'versioneer.py' to MANIFEST.in")
+        with open(manifest_in, "a") as f:
+            f.write("include versioneer.py\n")
+    else:
+        print(" 'versioneer.py' already in MANIFEST.in")
+    if cfg.versionfile_source not in simple_includes:
+        print(" appending versionfile_source ('%s') to MANIFEST.in" %
+              cfg.versionfile_source)
+        with open(manifest_in, "a") as f:
+            f.write("include %s\n" % cfg.versionfile_source)
+    else:
+        print(" versionfile_source already in MANIFEST.in")
 
-def get_cmdclass():
-    return {'version': cmd_version,
-            'update_files': cmd_update_files,
-            'build': cmd_build,
-            'sdist': cmd_sdist,
-            }
+    # Make VCS-specific changes. For git, this means creating/changing
+    # .gitattributes to mark _version.py for export-time keyword
+    # substitution.
+    do_vcs_install(manifest_in, cfg.versionfile_source, ipy)
+    return 0
+
+
+def scan_setup_py():
+    """Validate the contents of setup.py against Versioneer's expectations."""
+    found = set()
+    setters = False
+    errors = 0
+    with open("setup.py", "r") as f:
+        for line in f.readlines():
+            if "import versioneer" in line:
+                found.add("import")
+            if "versioneer.get_cmdclass()" in line:
+                found.add("cmdclass")
+            if "versioneer.get_version()" in line:
+                found.add("get_version")
+            if "versioneer.VCS" in line:
+                setters = True
+            if "versioneer.versionfile_source" in line:
+                setters = True
+    if len(found) != 3:
+        print("")
+        print("Your setup.py appears to be missing some important items")
+        print("(but I might be wrong). Please make sure it has something")
+        print("roughly like the following:")
+        print("")
+        print(" import versioneer")
+        print(" setup( version=versioneer.get_version(),")
+        print("        cmdclass=versioneer.get_cmdclass(),  ...)")
+        print("")
+        errors += 1
+    if setters:
+        print("You should remove lines like 'versioneer.VCS = ' and")
+        print("'versioneer.versionfile_source = ' . This configuration")
+        print("now lives in setup.cfg, and should be removed from setup.py")
+        print("")
+        errors += 1
+    return errors
+
+if __name__ == "__main__":
+    cmd = sys.argv[1]
+    if cmd == "setup":
+        errors = do_setup()
+        errors += scan_setup_py()
+        if errors:
+            sys.exit(1)
-- 
cgit v1.2.3


From 71b7fe430fea770b29e67dc5730a7c53afa2e6f6 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 1 Apr 2016 17:47:40 -0400
Subject: patch for multi-pkg repo

---
 common/versioneer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/versioneer.py b/common/versioneer.py
index 7ed2a21d..58339251 100644
--- a/common/versioneer.py
+++ b/common/versioneer.py
@@ -680,7 +680,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     expanded, and _version.py hasn't already been rewritten with a short
     version string, meaning we're inside a checked out source tree.
     """
-    if not os.path.exists(os.path.join(root, ".git")):
+    if not os.path.exists(os.path.join(root, "..", ".git")):
         if verbose:
             print("no .git in %%s" %% root)
         raise NotThisMethod("no .git directory")
@@ -1048,7 +1048,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     expanded, and _version.py hasn't already been rewritten with a short
     version string, meaning we're inside a checked out source tree.
     """
-    if not os.path.exists(os.path.join(root, ".git")):
+    if not os.path.exists(os.path.join(root, "..", ".git")):
         if verbose:
             print("no .git in %s" % root)
         raise NotThisMethod("no .git directory")
-- 
cgit v1.2.3


From b69ba40de308f06ae80f9847acbff40955fdee14 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 1 Apr 2016 17:51:08 -0400
Subject: [pkg] updated to versioneer 0.16 (patched)

---
 client/MANIFEST.in                         |    1 +
 client/setup.cfg                           |    7 +
 client/setup.py                            |   53 +-
 client/src/leap/soledad/client/_version.py |  548 ++++++--
 client/versioneer.py                       | 2063 +++++++++++++++++++++-------
 5 files changed, 2020 insertions(+), 652 deletions(-)

diff --git a/client/MANIFEST.in b/client/MANIFEST.in
index 7f6148ef..fc0ccfe8 100644
--- a/client/MANIFEST.in
+++ b/client/MANIFEST.in
@@ -2,3 +2,4 @@ include pkg/*
 include versioneer.py
 include LICENSE
 include CHANGELOG
+include src/leap/soledad/client/_version.py
diff --git a/client/setup.cfg b/client/setup.cfg
index 6b530888..d2d556d3 100644
--- a/client/setup.cfg
+++ b/client/setup.cfg
@@ -5,3 +5,10 @@ ignore = E731
 [flake8]
 exclude = versioneer.py,_version.py,ddocs.py,*.egg,build
 ignore = E731
+
+[versioneer]
+VCS = git
+style = pep440
+versionfile_source = src/leap/soledad/client/_version.py
+versionfile_build = leap/soledad/client/_version.py
+tag_prefix = 
diff --git a/client/setup.py b/client/setup.py
index dc555641..4480e247 100644
--- a/client/setup.py
+++ b/client/setup.py
@@ -21,15 +21,10 @@ import re
 from setuptools import setup
 from setuptools import find_packages
 from setuptools import Command
+import versioneer
 
 from pkg import utils
 
-import versioneer
-versioneer.versionfile_source = 'src/leap/soledad/client/_version.py'
-versioneer.versionfile_build = 'leap/soledad/client/_version.py'
-versioneer.tag_prefix = ''  # tags are like 1.2.0
-versioneer.parentdir_prefix = 'leap.soledad.client-'
-
 
 trove_classifiers = (
     "Development Status :: 3 - Alpha",
@@ -45,11 +40,11 @@ trove_classifiers = (
     "Topic :: Software Development :: Libraries :: Python Modules"
 )
 
-DOWNLOAD_BASE = ('https://github.com/leapcode/soledad/'
+DOWNLOAD_BASE = ('https://github.com/leapcode/bitmask_client/'
                  'archive/%s.tar.gz')
 _versions = versioneer.get_versions()
 VERSION = _versions['version']
-VERSION_FULL = _versions['full']
+VERSION_REVISION = _versions['full-revisionid']
 DOWNLOAD_URL = ""
 
 # get the short version for the download url
@@ -58,15 +53,30 @@ if len(_version_short) > 0:
     VERSION_SHORT = _version_short[0]
     DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT
 
-cmdclass = versioneer.get_cmdclass()
-
 
 class freeze_debianver(Command):
+
     """
     Freezes the version in a debian branch.
     To be used after merging the development branch onto the debian one.
     """
     user_options = []
+    template = r"""
+# This file was generated by the `freeze_debianver` command in setup.py
+# Using 'versioneer.py' (0.16) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+version_version = '{version}'
+full_revisionid = '{full_revisionid}'
+"""
+    templatefun = r"""
+
+def get_versions(default={}, verbose=False):
+        return {'version': version_version,
+                'full-revisionid': full_revisionid}
+"""
 
     def initialize_options(self):
         pass
@@ -80,28 +90,15 @@ class freeze_debianver(Command):
         if proceed != "y":
             print("He. You scared. Aborting.")
             return
-        template = r"""
-# This file was generated by the `freeze_debianver` command in setup.py
-# Using 'versioneer.py' (0.7+) from
-# revision-control system data, or from the parent directory name of an
-# unpacked source archive. Distribution tarballs contain a pre-generated copy
-# of this file.
-
-version_version = '{version}'
-version_full = '{version_full}'
-"""
-        templatefun = r"""
-
-def get_versions(default={}, verbose=False):
-        return {'version': version_version, 'full': version_full}
-"""
-        subst_template = template.format(
+        subst_template = self.template.format(
             version=VERSION_SHORT,
-            version_full=VERSION_FULL) + templatefun
-        with open(versioneer.versionfile_source, 'w') as f:
+            full_revisionid=VERSION_REVISION) + self.templatefun
+        versioneer_cfg = versioneer.get_config_from_root('.')
+        with open(versioneer_cfg.versionfile_source, 'w') as f:
             f.write(subst_template)
 
 
+cmdclass = versioneer.get_cmdclass()
 cmdclass["freeze_debianver"] = freeze_debianver
 
 
diff --git a/client/src/leap/soledad/client/_version.py b/client/src/leap/soledad/client/_version.py
index 588e4eb5..3ee3f81b 100644
--- a/client/src/leap/soledad/client/_version.py
+++ b/client/src/leap/soledad/client/_version.py
@@ -1,74 +1,157 @@
+
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
 
-# these strings will be replaced by git during git-archive
+"""Git implementation of _version.py."""
 
+import errno
+import os
+import re
 import subprocess
 import sys
-import re
-import os.path
 
-IN_LONG_VERSION_PY = True
 
-git_refnames = "$Format:%d$"
-git_full = "$Format:%H$"
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "$Format:%d$"
+    git_full = "$Format:%H$"
+    keywords = {"refnames": git_refnames, "full": git_full}
+    return keywords
 
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.cmd on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "pep440"
+    cfg.tag_prefix = ""
+    cfg.parentdir_prefix = "None"
+    cfg.versionfile_source = "src/leap/soledad/client/_version.py"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None
+    else:
         if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
+            print("unable to find command, tried %s" % (commands,))
         return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %s (error)" % args[0])
+            print("unable to run %s (error)" % dispcmd)
         return None
     return stdout
 
 
-def get_expanded_variables(versionfile_source):
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with "
+                  "prefix '%s'" % (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source, "r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {}  # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -84,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
         # "stabilization", as well as "HEAD" and "master".
         tags = set([r for r in refs if re.search(r'\d', r)])
         if verbose:
-            print("discarding '%s', no digits" % ",".join(refs - tags))
+            print("discarding '%s', no digits" % ",".join(refs-tags))
     if verbose:
         print("likely tags: %s" % ",".join(sorted(tags)))
     for ref in sorted(tags):
@@ -94,123 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             if verbose:
                 print("picking %s" % r)
             return {"version": r,
-                    "full": variables["full"].strip()}
-    # no suitable tags, so we use the full revision id
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return {"version": variables["full"].strip(),
-            "full": variables["full"].strip()}
-
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {}  # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(
-            os.path.join('..', here))
 
-    ######################################################
-    # XXX patch for our specific configuration with
-    # the three projects leap.soledad.{common, client, server}
-    # inside the same repo.
-    ######################################################
-    root = os.path.dirname(os.path.join('..', root))
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
 
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
     if not os.path.exists(os.path.join(root, ".git")):
         if verbose:
             print("no .git in %s" % root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.cmd"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" %
-                  (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source,
-                            verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {}  # without __file__, we have no hope
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%s*" % tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
+
+    try:
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
+
+    try:
+        root = os.path.realpath(__file__)
         # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
-    else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree"}
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start "
-                  "with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = ""
-parentdir_prefix = "leap.soledad.client-"
-versionfile_source = "src/leap/soledad/client/_version.py"
-
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = {"refnames": git_refnames, "full": git_full}
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version"}
diff --git a/client/versioneer.py b/client/versioneer.py
index 18dfd923..58339251 100644
--- a/client/versioneer.py
+++ b/client/versioneer.py
@@ -1,170 +1,641 @@
-#! /usr/bin/python
 
-"""versioneer.py
+# Version: 0.16
 
-(like a rocketeer, but for versions)
+"""The Versioneer - like a rocketeer, but for versions.
 
+The Versioneer
+==============
+
+* like a rocketeer, but for versions!
 * https://github.com/warner/python-versioneer
 * Brian Warner
 * License: Public Domain
-* Version: 0.7+
-
-This file helps distutils-based projects manage their version number by just
-creating version-control tags.
-
-For developers who work from a VCS-generated tree (e.g. 'git clone' etc),
-each 'setup.py version', 'setup.py build', 'setup.py sdist' will compute a
-version number by asking your version-control tool about the current
-checkout. The version number will be written into a generated _version.py
-file of your choosing, where it can be included by your __init__.py
-
-For users who work from a VCS-generated tarball (e.g. 'git archive'), it will
-compute a version number by looking at the name of the directory created when
-te tarball is unpacked. This conventionally includes both the name of the
-project and a version number.
-
-For users who work from a tarball built by 'setup.py sdist', it will get a
-version number from a previously-generated _version.py file.
-
-As a result, loading code directly from the source tree will not result in a
-real version. If you want real versions from VCS trees (where you frequently
-update from the upstream repository, or do new development), you will need to
-do a 'setup.py version' after each update, and load code from the build/
-directory.
-
-You need to provide this code with a few configuration values:
-
- versionfile_source:
-    A project-relative pathname into which the generated version strings
-    should be written. This is usually a _version.py next to your project's
-    main __init__.py file. If your project uses src/myproject/__init__.py,
-    this should be 'src/myproject/_version.py'. This file should be checked
-    in to your VCS as usual: the copy created below by 'setup.py
-    update_files' will include code that parses expanded VCS keywords in
-    generated tarballs. The 'build' and 'sdist' commands will replace it with
-    a copy that has just the calculated version string.
-
- versionfile_build:
-    Like versionfile_source, but relative to the build directory instead of
-    the source directory. These will differ when your setup.py uses
-    'package_dir='. If you have package_dir={'myproject': 'src/myproject'},
-    then you will probably have versionfile_build='myproject/_version.py' and
-    versionfile_source='src/myproject/_version.py'.
-
- tag_prefix: a string, like 'PROJECTNAME-', which appears at the start of all
-             VCS tags. If your tags look like 'myproject-1.2.0', then you
-             should use tag_prefix='myproject-'. If you use unprefixed tags
-             like '1.2.0', this should be an empty string.
-
- parentdir_prefix: a string, frequently the same as tag_prefix, which
-                   appears at the start of all unpacked tarball filenames. If
-                   your tarball unpacks into 'myproject-1.2.0', this should
-                   be 'myproject-'.
-
-To use it:
-
- 1: include this file in the top level of your project
- 2: make the following changes to the top of your setup.py:
-     import versioneer
-     versioneer.versionfile_source = 'src/myproject/_version.py'
-     versioneer.versionfile_build = 'myproject/_version.py'
-     versioneer.tag_prefix = '' # tags are like 1.2.0
-     versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0'
- 3: add the following arguments to the setup() call in your setup.py:
-     version=versioneer.get_version(),
-     cmdclass=versioneer.get_cmdclass(),
- 4: run 'setup.py update_files', which will create _version.py, and will
-    modify your __init__.py to define __version__ (by calling a function
-    from _version.py)
- 5: modify your MANIFEST.in to include versioneer.py
- 6: add both versioneer.py and the generated _version.py to your VCS
-"""
+* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy
+* [![Latest Version]
+(https://pypip.in/version/versioneer/badge.svg?style=flat)
+](https://pypi.python.org/pypi/versioneer/)
+* [![Build Status]
+(https://travis-ci.org/warner/python-versioneer.png?branch=master)
+](https://travis-ci.org/warner/python-versioneer)
+
+This is a tool for managing a recorded version number in distutils-based
+python projects. The goal is to remove the tedious and error-prone "update
+the embedded version string" step from your release process. Making a new
+release should be as easy as recording a new tag in your version-control
+system, and maybe making new tarballs.
+
+
+## Quick Install
+
+* `pip install versioneer` to somewhere to your $PATH
+* add a `[versioneer]` section to your setup.cfg (see below)
+* run `versioneer install` in your source tree, commit the results
+
+## Version Identifiers
+
+Source trees come from a variety of places:
+
+* a version-control system checkout (mostly used by developers)
+* a nightly tarball, produced by build automation
+* a snapshot tarball, produced by a web-based VCS browser, like github's
+  "tarball from tag" feature
+* a release tarball, produced by "setup.py sdist", distributed through PyPI
+
+Within each source tree, the version identifier (either a string or a number,
+this tool is format-agnostic) can come from a variety of places:
+
+* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows
+  about recent "tags" and an absolute revision-id
+* the name of the directory into which the tarball was unpacked
+* an expanded VCS keyword ($Id$, etc)
+* a `_version.py` created by some earlier build step
+
+For released software, the version identifier is closely related to a VCS
+tag. Some projects use tag names that include more than just the version
+string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool
+needs to strip the tag prefix to extract the version identifier. For
+unreleased software (between tags), the version identifier should provide
+enough information to help developers recreate the same tree, while also
+giving them an idea of roughly how old the tree is (after version 1.2, before
+version 1.3). Many VCS systems can report a description that captures this,
+for example `git describe --tags --dirty --always` reports things like
+"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the
+0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has
+uncommitted changes.
+
+The version identifier is used for multiple purposes:
+
+* to allow the module to self-identify its version: `myproject.__version__`
+* to choose a name and prefix for a 'setup.py sdist' tarball
+
+## Theory of Operation
+
+Versioneer works by adding a special `_version.py` file into your source
+tree, where your `__init__.py` can import it. This `_version.py` knows how to
+dynamically ask the VCS tool for version information at import time.
+
+`_version.py` also contains `$Revision$` markers, and the installation
+process marks `_version.py` to have this marker rewritten with a tag name
+during the `git archive` command. As a result, generated tarballs will
+contain enough information to get the proper version.
+
+To allow `setup.py` to compute a version too, a `versioneer.py` is added to
+the top level of your source tree, next to `setup.py` and the `setup.cfg`
+that configures it. This overrides several distutils/setuptools commands to
+compute the version when invoked, and changes `setup.py build` and `setup.py
+sdist` to replace `_version.py` with a small static file that contains just
+the generated version data.
+
+## Installation
+
+First, decide on values for the following configuration variables:
+
+* `VCS`: the version control system you use. Currently accepts "git".
+
+* `style`: the style of version string to be produced. See "Styles" below for
+  details. Defaults to "pep440", which looks like
+  `TAG[+DISTANCE.gSHORTHASH[.dirty]]`.
+
+* `versionfile_source`:
+
+  A project-relative pathname into which the generated version strings should
+  be written. This is usually a `_version.py` next to your project's main
+  `__init__.py` file, so it can be imported at runtime. If your project uses
+  `src/myproject/__init__.py`, this should be `src/myproject/_version.py`.
+  This file should be checked in to your VCS as usual: the copy created below
+  by `setup.py setup_versioneer` will include code that parses expanded VCS
+  keywords in generated tarballs. The 'build' and 'sdist' commands will
+  replace it with a copy that has just the calculated version string.
+
+  This must be set even if your project does not have any modules (and will
+  therefore never import `_version.py`), since "setup.py sdist" -based trees
+  still need somewhere to record the pre-calculated version strings. Anywhere
+  in the source tree should do. If there is a `__init__.py` next to your
+  `_version.py`, the `setup.py setup_versioneer` command (described below)
+  will append some `__version__`-setting assignments, if they aren't already
+  present.
+
+* `versionfile_build`:
+
+  Like `versionfile_source`, but relative to the build directory instead of
+  the source directory. These will differ when your setup.py uses
+  'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`,
+  then you will probably have `versionfile_build='myproject/_version.py'` and
+  `versionfile_source='src/myproject/_version.py'`.
+
+  If this is set to None, then `setup.py build` will not attempt to rewrite
+  any `_version.py` in the built tree. If your project does not have any
+  libraries (e.g. if it only builds a script), then you should use
+  `versionfile_build = None`. To actually use the computed version string,
+  your `setup.py` will need to override `distutils.command.build_scripts`
+  with a subclass that explicitly inserts a copy of
+  `versioneer.get_version()` into your script file. See
+  `test/demoapp-script-only/setup.py` for an example.
+
+* `tag_prefix`:
+
+  a string, like 'PROJECTNAME-', which appears at the start of all VCS tags.
+  If your tags look like 'myproject-1.2.0', then you should use
+  tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this
+  should be an empty string, using either `tag_prefix=` or `tag_prefix=''`.
+
+* `parentdir_prefix`:
+
+  a optional string, frequently the same as tag_prefix, which appears at the
+  start of all unpacked tarball filenames. If your tarball unpacks into
+  'myproject-1.2.0', this should be 'myproject-'. To disable this feature,
+  just omit the field from your `setup.cfg`.
+
+This tool provides one script, named `versioneer`. That script has one mode,
+"install", which writes a copy of `versioneer.py` into the current directory
+and runs `versioneer.py setup` to finish the installation.
+
+To versioneer-enable your project:
+
+* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and
+  populating it with the configuration values you decided earlier (note that
+  the option names are not case-sensitive):
+
+  ````
+  [versioneer]
+  VCS = git
+  style = pep440
+  versionfile_source = src/myproject/_version.py
+  versionfile_build = myproject/_version.py
+  tag_prefix =
+  parentdir_prefix = myproject-
+  ````
+
+* 2: Run `versioneer install`. This will do the following:
+
+  * copy `versioneer.py` into the top of your source tree
+  * create `_version.py` in the right place (`versionfile_source`)
+  * modify your `__init__.py` (if one exists next to `_version.py`) to define
+    `__version__` (by calling a function from `_version.py`)
+  * modify your `MANIFEST.in` to include both `versioneer.py` and the
+    generated `_version.py` in sdist tarballs
+
+  `versioneer install` will complain about any problems it finds with your
+  `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all
+  the problems.
+
+* 3: add a `import versioneer` to your setup.py, and add the following
+  arguments to the setup() call:
+
+        version=versioneer.get_version(),
+        cmdclass=versioneer.get_cmdclass(),
+
+* 4: commit these changes to your VCS. To make sure you won't forget,
+  `versioneer install` will mark everything it touched for addition using
+  `git add`. Don't forget to add `setup.py` and `setup.cfg` too.
+
+## Post-Installation Usage
+
+Once established, all uses of your tree from a VCS checkout should get the
+current version string. All generated tarballs should include an embedded
+version string (so users who unpack them will not need a VCS tool installed).
+
+If you distribute your project through PyPI, then the release process should
+boil down to two steps:
+
+* 1: git tag 1.0
+* 2: python setup.py register sdist upload
+
+If you distribute it through github (i.e. users use github to generate
+tarballs with `git archive`), the process is:
+
+* 1: git tag 1.0
+* 2: git push; git push --tags
+
+Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at
+least one tag in its history.
+
+## Version-String Flavors
+
+Code which uses Versioneer can learn about its version string at runtime by
+importing `_version` from your main `__init__.py` file and running the
+`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can
+import the top-level `versioneer.py` and run `get_versions()`.
+
+Both functions return a dictionary with different flavors of version
+information:
+
+* `['version']`: A condensed version string, rendered using the selected
+  style. This is the most commonly used value for the project's version
+  string. The default "pep440" style yields strings like `0.11`,
+  `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section
+  below for alternative styles.
+
+* `['full-revisionid']`: detailed revision identifier. For Git, this is the
+  full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac".
+
+* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that
+  this is only accurate if run in a VCS checkout, otherwise it is likely to
+  be False or None
+
+* `['error']`: if the version string could not be computed, this will be set
+  to a string describing the problem, otherwise it will be None. It may be
+  useful to throw an exception in setup.py if this is set, to avoid e.g.
+  creating tarballs with a version string of "unknown".
+
+Some variants are more useful than others. Including `full-revisionid` in a
+bug report should allow developers to reconstruct the exact code being tested
+(or indicate the presence of local changes that should be shared with the
+developers). `version` is suitable for display in an "about" box or a CLI
+`--version` output: it can be easily compared against release notes and lists
+of bugs fixed in various releases.
+
+The installer adds the following text to your `__init__.py` to place a basic
+version in `YOURPROJECT.__version__`:
+
+    from ._version import get_versions
+    __version__ = get_versions()['version']
+    del get_versions
+
+## Styles
+
+The setup.cfg `style=` configuration controls how the VCS information is
+rendered into a version string.
+
+The default style, "pep440", produces a PEP440-compliant string, equal to the
+un-prefixed tag name for actual releases, and containing an additional "local
+version" section with more detail for in-between builds. For Git, this is
+TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags
+--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the
+tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and
+that this commit is two revisions ("+2") beyond the "0.11" tag. For released
+software (exactly equal to a known tag), the identifier will only contain the
+stripped tag, e.g. "0.11".
+
+Other styles are available. See details.md in the Versioneer source tree for
+descriptions.
+
+## Debugging
+
+Versioneer tries to avoid fatal errors: if something goes wrong, it will tend
+to return a version of "0+unknown". To investigate the problem, run `setup.py
+version`, which will run the version-lookup code in a verbose mode, and will
+display the full contents of `get_versions()` (including the `error` string,
+which may help identify what went wrong).
+
+## Updating Versioneer
+
+To upgrade your project to a new release of Versioneer, do the following:
+
+* install the new Versioneer (`pip install -U versioneer` or equivalent)
+* edit `setup.cfg`, if necessary, to include any new configuration settings
+  indicated by the release notes
+* re-run `versioneer install` in your source tree, to replace
+  `SRC/_version.py`
+* commit any changed files
+
+### Upgrading to 0.16
+
+Nothing special.
+
+### Upgrading to 0.15
+
+Starting with this version, Versioneer is configured with a `[versioneer]`
+section in your `setup.cfg` file. Earlier versions required the `setup.py` to
+set attributes on the `versioneer` module immediately after import. The new
+version will refuse to run (raising an exception during import) until you
+have provided the necessary `setup.cfg` section.
+
+In addition, the Versioneer package provides an executable named
+`versioneer`, and the installation process is driven by running `versioneer
+install`. In 0.14 and earlier, the executable was named
+`versioneer-installer` and was run without an argument.
+
+### Upgrading to 0.14
 
-import os, sys, re
-from distutils.core import Command
-from distutils.command.sdist import sdist as _sdist
-from distutils.command.build import build as _build
+0.14 changes the format of the version string. 0.13 and earlier used
+hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a
+plus-separated "local version" section strings, with dot-separated
+components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old
+format, but should be ok with the new one.
 
-versionfile_source = None
-versionfile_build = None
-tag_prefix = None
-parentdir_prefix = None
+### Upgrading from 0.11 to 0.12
 
-VCS = "git"
-IN_LONG_VERSION_PY = False
+Nothing special.
 
+### Upgrading from 0.10 to 0.11
 
-LONG_VERSION_PY = '''
-IN_LONG_VERSION_PY = True
+You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running
+`setup.py setup_versioneer`. This will enable the use of additional
+version-control systems (SVN, etc) in the future.
+
+## Future Directions
+
+This tool is designed to make it easily extended to other version-control
+systems: all VCS-specific components are in separate directories like
+src/git/ . The top-level `versioneer.py` script is assembled from these
+components by running make-versioneer.py . In the future, make-versioneer.py
+will take a VCS name as an argument, and will construct a version of
+`versioneer.py` that is specific to the given VCS. It might also take the
+configuration arguments that are currently provided manually during
+installation by editing setup.py . Alternatively, it might go the other
+direction and include code from all supported VCS systems, reducing the
+number of intermediate scripts.
+
+
+## License
+
+To make Versioneer easier to embed, all its code is dedicated to the public
+domain. The `_version.py` that it creates is also in the public domain.
+Specifically, both are released under the Creative Commons "Public Domain
+Dedication" license (CC0-1.0), as described in
+https://creativecommons.org/publicdomain/zero/1.0/ .
+
+"""
+
+from __future__ import print_function
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+import errno
+import json
+import os
+import re
+import subprocess
+import sys
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_root():
+    """Get the project root directory.
+
+    We require that all commands are run from the project root, i.e. the
+    directory that contains setup.py, setup.cfg, and versioneer.py .
+    """
+    root = os.path.realpath(os.path.abspath(os.getcwd()))
+    setup_py = os.path.join(root, "setup.py")
+    versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        # allow 'python path/to/setup.py COMMAND'
+        root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
+        setup_py = os.path.join(root, "setup.py")
+        versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        err = ("Versioneer was unable to run the project root directory. "
+               "Versioneer requires setup.py to be executed from "
+               "its immediate directory (like 'python setup.py COMMAND'), "
+               "or in a way that lets it use sys.argv[0] to find the root "
+               "(like 'python path/to/setup.py COMMAND').")
+        raise VersioneerBadRootError(err)
+    try:
+        # Certain runtime workflows (setup.py install/develop in a setuptools
+        # tree) execute all dependencies in a single python process, so
+        # "versioneer" may be imported multiple times, and python's shared
+        # module-import table will cache the first one. So we can't use
+        # os.path.dirname(__file__), as that will find whichever
+        # versioneer.py was first imported, even in later projects.
+        me = os.path.realpath(os.path.abspath(__file__))
+        if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]:
+            print("Warning: build in %s is using versioneer.py from %s"
+                  % (os.path.dirname(me), versioneer_py))
+    except NameError:
+        pass
+    return root
+
+
+def get_config_from_root(root):
+    """Read the project setup.cfg file to determine Versioneer config."""
+    # This might raise EnvironmentError (if setup.cfg is missing), or
+    # configparser.NoSectionError (if it lacks a [versioneer] section), or
+    # configparser.NoOptionError (if it lacks "VCS="). See the docstring at
+    # the top of versioneer.py for instructions on writing your setup.cfg .
+    setup_cfg = os.path.join(root, "setup.cfg")
+    parser = configparser.SafeConfigParser()
+    with open(setup_cfg, "r") as f:
+        parser.readfp(f)
+    VCS = parser.get("versioneer", "VCS")  # mandatory
+
+    def get(parser, name):
+        if parser.has_option("versioneer", name):
+            return parser.get("versioneer", name)
+        return None
+    cfg = VersioneerConfig()
+    cfg.VCS = VCS
+    cfg.style = get(parser, "style") or ""
+    cfg.versionfile_source = get(parser, "versionfile_source")
+    cfg.versionfile_build = get(parser, "versionfile_build")
+    cfg.tag_prefix = get(parser, "tag_prefix")
+    if cfg.tag_prefix in ("''", '""'):
+        cfg.tag_prefix = ""
+    cfg.parentdir_prefix = get(parser, "parentdir_prefix")
+    cfg.verbose = get(parser, "verbose")
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+# these dictionaries contain VCS-specific tools
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None
+    else:
+        if verbose:
+            print("unable to find command, tried %s" % (commands,))
+        return None
+    stdout = p.communicate()[0].strip()
+    if sys.version_info[0] >= 3:
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %s (error)" % dispcmd)
+        return None
+    return stdout
+LONG_VERSION_PY['git'] = '''
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
-
-# these strings will be replaced by git during git-archive
-git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
-git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
 
+"""Git implementation of _version.py."""
 
+import errno
+import os
+import re
 import subprocess
 import sys
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.exe on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
+    git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+    keywords = {"refnames": git_refnames, "full": git_full}
+    return keywords
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "%(STYLE)s"
+    cfg.tag_prefix = "%(TAG_PREFIX)s"
+    cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s"
+    cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %%s" %% dispcmd)
+                print(e)
+            return None
+    else:
         if verbose:
-            print("unable to run %%s" %% args[0])
-            print(e)
+            print("unable to find command, tried %%s" %% (commands,))
         return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %%s (error)" %% args[0])
+            print("unable to run %%s (error)" %% dispcmd)
         return None
     return stdout
 
 
-import sys
-import re
-import os.path
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%%s', but '%%s' doesn't start with "
+                  "prefix '%%s'" %% (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
+
 
-def get_expanded_variables(versionfile_source):
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source,"r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -189,172 +660,350 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %%s" %% r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
-    if not os.path.exists(os.path.join(root, ".git")):
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
+    if not os.path.exists(os.path.join(root, "..", ".git")):
         if verbose:
             print("no .git in %%s" %% root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
-        # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%%s*" %% tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%%s'"
+                               %% describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%%s' doesn't start with prefix '%%s'"
+                print(fmt %% (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'"
+                               %% (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
     else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+        # exception #1
+        rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%%d" %% pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%%d" %% pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%%s" %% pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%%s" %% pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %%
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = "%(TAG_PREFIX)s"
-parentdir_prefix = "%(PARENTDIR_PREFIX)s"
-versionfile_source = "%(VERSIONFILE_SOURCE)s"
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = { "refnames": git_refnames, "full": git_full }
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
 
-'''
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
 
+    Like 'git describe --tags --dirty --always'.
 
-import subprocess
-import sys
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%%s'" %% style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
 
-def run_command(args, cwd=None, verbose=False):
     try:
-        # remember shell=False, so use git.exe on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
-        if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
-        return None
-    stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
-        stdout = stdout.decode()
-    if p.returncode != 0:
-        if verbose:
-            print("unable to run %s (error)" % args[0])
-        return None
-    return stdout
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
 
+    try:
+        root = os.path.realpath(__file__)
+        # versionfile_source is the relative path from the top of the source
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
+            root = os.path.dirname(root)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree"}
 
-import sys
-import re
-import os.path
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
 
-def get_expanded_variables(versionfile_source):
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version"}
+'''
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source,"r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -379,117 +1028,122 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %s" % r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = os.path.dirname(
-        os.path.join('..', here))
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(
-            os.path.join('..', here))
 
-    ######################################################
-    # XXX patch for our specific configuration with
-    # the three projects leap.soledad.{common, client, server}
-    # inside the same repo.
-    ######################################################
-    root = os.path.dirname(os.path.join('..', root))
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
 
-    if not os.path.exists(os.path.join(root, ".git")):
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
+    if not os.path.exists(os.path.join(root, "..", ".git")):
         if verbose:
             print("no .git in %s" % root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(os.path.join('..', __file__))
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
-        # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%s*" % tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
     else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
+    return pieces
 
-import sys
 
-def do_vcs_install(versionfile_source, ipy):
-    GIT = "git"
+def do_vcs_install(manifest_in, versionfile_source, ipy):
+    """Git-specific installation logic for Versioneer.
+
+    For Git, this means creating/changing .gitattributes to mark _version.py
+    for export-time keyword substitution.
+    """
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    run_command([GIT, "add", "versioneer.py"])
-    run_command([GIT, "add", versionfile_source])
-    run_command([GIT, "add", ipy])
+        GITS = ["git.cmd", "git.exe"]
+    files = [manifest_in, versionfile_source]
+    if ipy:
+        files.append(ipy)
+    try:
+        me = __file__
+        if me.endswith(".pyc") or me.endswith(".pyo"):
+            me = os.path.splitext(me)[0] + ".py"
+        versioneer_file = os.path.relpath(me)
+    except NameError:
+        versioneer_file = "versioneer.py"
+    files.append(versioneer_file)
     present = False
     try:
         f = open(".gitattributes", "r")
@@ -504,135 +1158,487 @@ def do_vcs_install(versionfile_source, ipy):
         f = open(".gitattributes", "a+")
         f.write("%s export-subst\n" % versionfile_source)
         f.close()
-        run_command([GIT, "add", ".gitattributes"])
+        files.append(".gitattributes")
+    run_command(GITS, ["add", "--"] + files)
+
 
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with "
+                  "prefix '%s'" % (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
 
 SHORT_VERSION_PY = """
-# This file was generated by 'versioneer.py' (0.7+) from
+# This file was generated by 'versioneer.py' (0.16) from
 # revision-control system data, or from the parent directory name of an
 # unpacked source archive. Distribution tarballs contain a pre-generated copy
 # of this file.
 
-version_version = '%(version)s'
-version_full = '%(full)s'
-def get_versions(default={}, verbose=False):
-    return {'version': version_version, 'full': version_full}
+import json
+import sys
+
+version_json = '''
+%s
+'''  # END VERSION_JSON
 
+
+def get_versions():
+    return json.loads(version_json)
 """
 
-DEFAULT = {"version": "unknown", "full": "unknown"}
 
 def versions_from_file(filename):
-    versions = {}
+    """Try to determine the version from _version.py if present."""
     try:
-        f = open(filename)
+        with open(filename) as f:
+            contents = f.read()
     except EnvironmentError:
-        return versions
-    for line in f.readlines():
-        mo = re.match("version_version = '([^']+)'", line)
-        if mo:
-            versions["version"] = mo.group(1)
-        mo = re.match("version_full = '([^']+)'", line)
-        if mo:
-            versions["full"] = mo.group(1)
-    f.close()
-    return versions
+        raise NotThisMethod("unable to read _version.py")
+    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
+                   contents, re.M | re.S)
+    if not mo:
+        raise NotThisMethod("no version_json in _version.py")
+    return json.loads(mo.group(1))
+
 
 def write_to_version_file(filename, versions):
-    f = open(filename, "w")
-    f.write(SHORT_VERSION_PY % versions)
-    f.close()
+    """Write the given version number to the given _version.py file."""
+    os.unlink(filename)
+    contents = json.dumps(versions, sort_keys=True,
+                          indent=1, separators=(",", ": "))
+    with open(filename, "w") as f:
+        f.write(SHORT_VERSION_PY % contents)
+
     print("set %s to '%s'" % (filename, versions["version"]))
 
 
-def get_best_versions(versionfile, tag_prefix, parentdir_prefix,
-                      default=DEFAULT, verbose=False):
-    # returns dict with two keys: 'version' and 'full'
-    #
-    # extract version from first of _version.py, 'git describe', parentdir.
-    # This is meant to work for developers using a source checkout, for users
-    # of a tarball created by 'setup.py sdist', and for users of a
-    # tarball/zipball created by 'git archive' or github's download-from-tag
-    # feature.
-
-    variables = get_expanded_variables(versionfile_source)
-    if variables:
-        ver = versions_from_expanded_variables(variables, tag_prefix)
-        if ver:
-            if verbose: print("got version from expanded variable %s" % ver)
-            return ver
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
 
-    ver = versions_from_file(versionfile)
-    if ver:
-        if verbose: print("got version from file %s %s" % (versionfile, ver))
-        return ver
 
-    ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if ver:
-        if verbose: print("got version from git %s" % ver)
-        return ver
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
 
-    ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose)
-    if ver:
-        if verbose: print("got version from parentdir %s" % ver)
-        return ver
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
 
-    if verbose: print("got version from default %s" % ver)
-    return default
-
-def get_versions(default=DEFAULT, verbose=False):
-    assert versionfile_source is not None, "please set versioneer.versionfile_source"
-    assert tag_prefix is not None, "please set versioneer.tag_prefix"
-    assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix"
-    return get_best_versions(versionfile_source, tag_prefix, parentdir_prefix,
-                             default=default, verbose=verbose)
-def get_version(verbose=False):
-    return get_versions(verbose=verbose)["version"]
-
-class cmd_version(Command):
-    description = "report generated version string"
-    user_options = []
-    boolean_options = []
-    def initialize_options(self):
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+class VersioneerBadRootError(Exception):
+    """The project root directory is unknown or missing key files."""
+
+
+def get_versions(verbose=False):
+    """Get the project version from whatever source is available.
+
+    Returns dict with two keys: 'version' and 'full'.
+    """
+    if "versioneer" in sys.modules:
+        # see the discussion in cmdclass.py:get_cmdclass()
+        del sys.modules["versioneer"]
+
+    root = get_root()
+    cfg = get_config_from_root(root)
+
+    assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg"
+    handlers = HANDLERS.get(cfg.VCS)
+    assert handlers, "unrecognized VCS '%s'" % cfg.VCS
+    verbose = verbose or cfg.verbose
+    assert cfg.versionfile_source is not None, \
+        "please set versioneer.versionfile_source"
+    assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix"
+
+    versionfile_abs = os.path.join(root, cfg.versionfile_source)
+
+    # extract version from first of: _version.py, VCS command (e.g. 'git
+    # describe'), parentdir. This is meant to work for developers using a
+    # source checkout, for users of a tarball created by 'setup.py sdist',
+    # and for users of a tarball/zipball created by 'git archive' or github's
+    # download-from-tag feature or the equivalent in other VCSes.
+
+    get_keywords_f = handlers.get("get_keywords")
+    from_keywords_f = handlers.get("keywords")
+    if get_keywords_f and from_keywords_f:
+        try:
+            keywords = get_keywords_f(versionfile_abs)
+            ver = from_keywords_f(keywords, cfg.tag_prefix, verbose)
+            if verbose:
+                print("got version from expanded keyword %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        ver = versions_from_file(versionfile_abs)
+        if verbose:
+            print("got version from file %s %s" % (versionfile_abs, ver))
+        return ver
+    except NotThisMethod:
         pass
-    def finalize_options(self):
+
+    from_vcs_f = handlers.get("pieces_from_vcs")
+    if from_vcs_f:
+        try:
+            pieces = from_vcs_f(cfg.tag_prefix, root, verbose)
+            ver = render(pieces, cfg.style)
+            if verbose:
+                print("got version from VCS %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        if cfg.parentdir_prefix:
+            ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+            if verbose:
+                print("got version from parentdir %s" % ver)
+            return ver
+    except NotThisMethod:
         pass
-    def run(self):
-        ver = get_version(verbose=True)
-        print("Version is currently: %s" % ver)
-
-
-class cmd_build(_build):
-    def run(self):
-        versions = get_versions(verbose=True)
-        _build.run(self)
-        # now locate _version.py in the new build/ directory and replace it
-        # with an updated value
-        target_versionfile = os.path.join(self.build_lib, versionfile_build)
-        print("UPDATING %s" % target_versionfile)
-        os.unlink(target_versionfile)
-        f = open(target_versionfile, "w")
-        f.write(SHORT_VERSION_PY % versions)
-        f.close()
 
-class cmd_sdist(_sdist):
-    def run(self):
-        versions = get_versions(verbose=True)
-        self._versioneer_generated_versions = versions
-        # unless we update this, the command will keep using the old version
-        self.distribution.metadata.version = versions["version"]
-        return _sdist.run(self)
-
-    def make_release_tree(self, base_dir, files):
-        _sdist.make_release_tree(self, base_dir, files)
-        # now locate _version.py in the new base_dir directory (remembering
-        # that it may be a hardlink) and replace it with an updated value
-        target_versionfile = os.path.join(base_dir, versionfile_source)
-        print("UPDATING %s" % target_versionfile)
-        os.unlink(target_versionfile)
-        f = open(target_versionfile, "w")
-        f.write(SHORT_VERSION_PY % self._versioneer_generated_versions)
-        f.close()
+    if verbose:
+        print("unable to compute version")
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None, "error": "unable to compute version"}
+
+
+def get_version():
+    """Get the short version string for this project."""
+    return get_versions()["version"]
+
+
+def get_cmdclass():
+    """Get the custom setuptools/distutils subclasses used by Versioneer."""
+    if "versioneer" in sys.modules:
+        del sys.modules["versioneer"]
+        # this fixes the "python setup.py develop" case (also 'install' and
+        # 'easy_install .'), in which subdependencies of the main project are
+        # built (using setup.py bdist_egg) in the same python process. Assume
+        # a main project A and a dependency B, which use different versions
+        # of Versioneer. A's setup.py imports A's Versioneer, leaving it in
+        # sys.modules by the time B's setup.py is executed, causing B to run
+        # with the wrong versioneer. Setuptools wraps the sub-dep builds in a
+        # sandbox that restores sys.modules to it's pre-build state, so the
+        # parent is protected against the child's "import versioneer". By
+        # removing ourselves from sys.modules here, before the child build
+        # happens, we protect the child from the parent's versioneer too.
+        # Also see https://github.com/warner/python-versioneer/issues/52
+
+    cmds = {}
+
+    # we add "version" to both distutils and setuptools
+    from distutils.core import Command
+
+    class cmd_version(Command):
+        description = "report generated version string"
+        user_options = []
+        boolean_options = []
+
+        def initialize_options(self):
+            pass
+
+        def finalize_options(self):
+            pass
+
+        def run(self):
+            vers = get_versions(verbose=True)
+            print("Version: %s" % vers["version"])
+            print(" full-revisionid: %s" % vers.get("full-revisionid"))
+            print(" dirty: %s" % vers.get("dirty"))
+            if vers["error"]:
+                print(" error: %s" % vers["error"])
+    cmds["version"] = cmd_version
+
+    # we override "build_py" in both distutils and setuptools
+    #
+    # most invocation pathways end up running build_py:
+    #  distutils/build -> build_py
+    #  distutils/install -> distutils/build ->..
+    #  setuptools/bdist_wheel -> distutils/install ->..
+    #  setuptools/bdist_egg -> distutils/install_lib -> build_py
+    #  setuptools/install -> bdist_egg ->..
+    #  setuptools/develop -> ?
+
+    # we override different "build_py" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.build_py import build_py as _build_py
+    else:
+        from distutils.command.build_py import build_py as _build_py
+
+    class cmd_build_py(_build_py):
+        def run(self):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            versions = get_versions()
+            _build_py.run(self)
+            # now locate _version.py in the new build/ directory and replace
+            # it with an updated value
+            if cfg.versionfile_build:
+                target_versionfile = os.path.join(self.build_lib,
+                                                  cfg.versionfile_build)
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+    cmds["build_py"] = cmd_build_py
+
+    if "cx_Freeze" in sys.modules:  # cx_freeze enabled?
+        from cx_Freeze.dist import build_exe as _build_exe
+
+        class cmd_build_exe(_build_exe):
+            def run(self):
+                root = get_root()
+                cfg = get_config_from_root(root)
+                versions = get_versions()
+                target_versionfile = cfg.versionfile_source
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+
+                _build_exe.run(self)
+                os.unlink(target_versionfile)
+                with open(cfg.versionfile_source, "w") as f:
+                    LONG = LONG_VERSION_PY[cfg.VCS]
+                    f.write(LONG %
+                            {"DOLLAR": "$",
+                             "STYLE": cfg.style,
+                             "TAG_PREFIX": cfg.tag_prefix,
+                             "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                             "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                             })
+        cmds["build_exe"] = cmd_build_exe
+        del cmds["build_py"]
+
+    # we override different "sdist" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.sdist import sdist as _sdist
+    else:
+        from distutils.command.sdist import sdist as _sdist
+
+    class cmd_sdist(_sdist):
+        def run(self):
+            versions = get_versions()
+            self._versioneer_generated_versions = versions
+            # unless we update this, the command will keep using the old
+            # version
+            self.distribution.metadata.version = versions["version"]
+            return _sdist.run(self)
+
+        def make_release_tree(self, base_dir, files):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            _sdist.make_release_tree(self, base_dir, files)
+            # now locate _version.py in the new base_dir directory
+            # (remembering that it may be a hardlink) and replace it with an
+            # updated value
+            target_versionfile = os.path.join(base_dir, cfg.versionfile_source)
+            print("UPDATING %s" % target_versionfile)
+            write_to_version_file(target_versionfile,
+                                  self._versioneer_generated_versions)
+    cmds["sdist"] = cmd_sdist
+
+    return cmds
+
+
+CONFIG_ERROR = """
+setup.cfg is missing the necessary Versioneer configuration. You need
+a section like:
+
+ [versioneer]
+ VCS = git
+ style = pep440
+ versionfile_source = src/myproject/_version.py
+ versionfile_build = myproject/_version.py
+ tag_prefix =
+ parentdir_prefix = myproject-
+
+You will also need to edit your setup.py to use the results:
+
+ import versioneer
+ setup(version=versioneer.get_version(),
+       cmdclass=versioneer.get_cmdclass(), ...)
+
+Please read the docstring in ./versioneer.py for configuration instructions,
+edit setup.cfg, and re-run the installer or 'python versioneer.py setup'.
+"""
+
+SAMPLE_CONFIG = """
+# See the docstring in versioneer.py for instructions. Note that you must
+# re-run 'versioneer.py setup' after changing this section, and commit the
+# resulting files.
+
+[versioneer]
+#VCS = git
+#style = pep440
+#versionfile_source =
+#versionfile_build =
+#tag_prefix =
+#parentdir_prefix =
+
+"""
 
 INIT_PY_SNIPPET = """
 from ._version import get_versions
@@ -640,40 +1646,129 @@ __version__ = get_versions()['version']
 del get_versions
 """
 
-class cmd_update_files(Command):
-    description = "modify __init__.py and create _version.py"
-    user_options = []
-    boolean_options = []
-    def initialize_options(self):
-        pass
-    def finalize_options(self):
-        pass
-    def run(self):
-        ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py")
-        print(" creating %s" % versionfile_source)
-        f = open(versionfile_source, "w")
-        f.write(LONG_VERSION_PY % {"DOLLAR": "$",
-                                   "TAG_PREFIX": tag_prefix,
-                                   "PARENTDIR_PREFIX": parentdir_prefix,
-                                   "VERSIONFILE_SOURCE": versionfile_source,
-                                   })
-        f.close()
+
+def do_setup():
+    """Main VCS-independent setup function for installing Versioneer."""
+    root = get_root()
+    try:
+        cfg = get_config_from_root(root)
+    except (EnvironmentError, configparser.NoSectionError,
+            configparser.NoOptionError) as e:
+        if isinstance(e, (EnvironmentError, configparser.NoSectionError)):
+            print("Adding sample versioneer config to setup.cfg",
+                  file=sys.stderr)
+            with open(os.path.join(root, "setup.cfg"), "a") as f:
+                f.write(SAMPLE_CONFIG)
+        print(CONFIG_ERROR, file=sys.stderr)
+        return 1
+
+    print(" creating %s" % cfg.versionfile_source)
+    with open(cfg.versionfile_source, "w") as f:
+        LONG = LONG_VERSION_PY[cfg.VCS]
+        f.write(LONG % {"DOLLAR": "$",
+                        "STYLE": cfg.style,
+                        "TAG_PREFIX": cfg.tag_prefix,
+                        "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                        "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                        })
+
+    ipy = os.path.join(os.path.dirname(cfg.versionfile_source),
+                       "__init__.py")
+    if os.path.exists(ipy):
         try:
-            old = open(ipy, "r").read()
+            with open(ipy, "r") as f:
+                old = f.read()
         except EnvironmentError:
             old = ""
         if INIT_PY_SNIPPET not in old:
             print(" appending to %s" % ipy)
-            f = open(ipy, "a")
-            f.write(INIT_PY_SNIPPET)
-            f.close()
+            with open(ipy, "a") as f:
+                f.write(INIT_PY_SNIPPET)
         else:
             print(" %s unmodified" % ipy)
-        do_vcs_install(versionfile_source, ipy)
+    else:
+        print(" %s doesn't exist, ok" % ipy)
+        ipy = None
+
+    # Make sure both the top-level "versioneer.py" and versionfile_source
+    # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so
+    # they'll be copied into source distributions. Pip won't be able to
+    # install the package without this.
+    manifest_in = os.path.join(root, "MANIFEST.in")
+    simple_includes = set()
+    try:
+        with open(manifest_in, "r") as f:
+            for line in f:
+                if line.startswith("include "):
+                    for include in line.split()[1:]:
+                        simple_includes.add(include)
+    except EnvironmentError:
+        pass
+    # That doesn't cover everything MANIFEST.in can do
+    # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
+    # it might give some false negatives. Appending redundant 'include'
+    # lines is safe, though.
+    if "versioneer.py" not in simple_includes:
+        print(" appending 'versioneer.py' to MANIFEST.in")
+        with open(manifest_in, "a") as f:
+            f.write("include versioneer.py\n")
+    else:
+        print(" 'versioneer.py' already in MANIFEST.in")
+    if cfg.versionfile_source not in simple_includes:
+        print(" appending versionfile_source ('%s') to MANIFEST.in" %
+              cfg.versionfile_source)
+        with open(manifest_in, "a") as f:
+            f.write("include %s\n" % cfg.versionfile_source)
+    else:
+        print(" versionfile_source already in MANIFEST.in")
 
-def get_cmdclass():
-    return {'version': cmd_version,
-            'update_files': cmd_update_files,
-            'build': cmd_build,
-            'sdist': cmd_sdist,
-            }
+    # Make VCS-specific changes. For git, this means creating/changing
+    # .gitattributes to mark _version.py for export-time keyword
+    # substitution.
+    do_vcs_install(manifest_in, cfg.versionfile_source, ipy)
+    return 0
+
+
+def scan_setup_py():
+    """Validate the contents of setup.py against Versioneer's expectations."""
+    found = set()
+    setters = False
+    errors = 0
+    with open("setup.py", "r") as f:
+        for line in f.readlines():
+            if "import versioneer" in line:
+                found.add("import")
+            if "versioneer.get_cmdclass()" in line:
+                found.add("cmdclass")
+            if "versioneer.get_version()" in line:
+                found.add("get_version")
+            if "versioneer.VCS" in line:
+                setters = True
+            if "versioneer.versionfile_source" in line:
+                setters = True
+    if len(found) != 3:
+        print("")
+        print("Your setup.py appears to be missing some important items")
+        print("(but I might be wrong). Please make sure it has something")
+        print("roughly like the following:")
+        print("")
+        print(" import versioneer")
+        print(" setup( version=versioneer.get_version(),")
+        print("        cmdclass=versioneer.get_cmdclass(),  ...)")
+        print("")
+        errors += 1
+    if setters:
+        print("You should remove lines like 'versioneer.VCS = ' and")
+        print("'versioneer.versionfile_source = ' . This configuration")
+        print("now lives in setup.cfg, and should be removed from setup.py")
+        print("")
+        errors += 1
+    return errors
+
+if __name__ == "__main__":
+    cmd = sys.argv[1]
+    if cmd == "setup":
+        errors = do_setup()
+        errors += scan_setup_py()
+        if errors:
+            sys.exit(1)
-- 
cgit v1.2.3


From b5fd4060831b82f1e4ea26d6ef3792793be76d77 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 1 Apr 2016 17:54:58 -0400
Subject: [pkg] updated to versioneer 0.16 (patched)

---
 server/MANIFEST.in                         |    1 +
 server/setup.cfg                           |    7 +
 server/setup.py                            |   52 +-
 server/src/leap/soledad/server/_version.py |  548 ++++++--
 server/versioneer.py                       | 2063 +++++++++++++++++++++-------
 5 files changed, 2020 insertions(+), 651 deletions(-)

diff --git a/server/MANIFEST.in b/server/MANIFEST.in
index 7f6148ef..f047f3ee 100644
--- a/server/MANIFEST.in
+++ b/server/MANIFEST.in
@@ -2,3 +2,4 @@ include pkg/*
 include versioneer.py
 include LICENSE
 include CHANGELOG
+include src/leap/soledad/server/_version.py
diff --git a/server/setup.cfg b/server/setup.cfg
index 6b530888..af7c5280 100644
--- a/server/setup.cfg
+++ b/server/setup.cfg
@@ -5,3 +5,10 @@ ignore = E731
 [flake8]
 exclude = versioneer.py,_version.py,ddocs.py,*.egg,build
 ignore = E731
+
+[versioneer]
+VCS = git
+style = pep440
+versionfile_source = src/leap/soledad/server/_version.py
+versionfile_build = leap/soledad/server/_version.py
+tag_prefix = 
diff --git a/server/setup.py b/server/setup.py
index b3942551..8a7fbe45 100644
--- a/server/setup.py
+++ b/server/setup.py
@@ -22,14 +22,10 @@ import re
 from setuptools import setup
 from setuptools import find_packages
 from setuptools import Command
+import versioneer
 
 from pkg import utils
 
-import versioneer
-versioneer.versionfile_source = 'src/leap/soledad/server/_version.py'
-versioneer.versionfile_build = 'leap/soledad/server/_version.py'
-versioneer.tag_prefix = ''  # tags are like 1.2.0
-versioneer.parentdir_prefix = 'leap.soledad.server-'
 
 isset = lambda var: os.environ.get(var, None)
 if isset('VIRTUAL_ENV') or isset('LEAP_SKIP_INIT'):
@@ -53,11 +49,11 @@ trove_classifiers = (
     "Topic :: Software Development :: Libraries :: Python Modules"
 )
 
-DOWNLOAD_BASE = ('https://github.com/leapcode/soledad/'
+DOWNLOAD_BASE = ('https://github.com/leapcode/bitmask_client/'
                  'archive/%s.tar.gz')
 _versions = versioneer.get_versions()
 VERSION = _versions['version']
-VERSION_FULL = _versions['full']
+VERSION_REVISION = _versions['full-revisionid']
 DOWNLOAD_URL = ""
 
 # get the short version for the download url
@@ -66,15 +62,30 @@ if len(_version_short) > 0:
     VERSION_SHORT = _version_short[0]
     DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT
 
-cmdclass = versioneer.get_cmdclass()
-
 
 class freeze_debianver(Command):
+
     """
     Freezes the version in a debian branch.
     To be used after merging the development branch onto the debian one.
     """
     user_options = []
+    template = r"""
+# This file was generated by the `freeze_debianver` command in setup.py
+# Using 'versioneer.py' (0.16) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+version_version = '{version}'
+full_revisionid = '{full_revisionid}'
+"""
+    templatefun = r"""
+
+def get_versions(default={}, verbose=False):
+        return {'version': version_version,
+                'full-revisionid': full_revisionid}
+"""
 
     def initialize_options(self):
         pass
@@ -88,28 +99,15 @@ class freeze_debianver(Command):
         if proceed != "y":
             print("He. You scared. Aborting.")
             return
-        template = r"""
-# This file was generated by the `freeze_debianver` command in setup.py
-# Using 'versioneer.py' (0.7+) from
-# revision-control system data, or from the parent directory name of an
-# unpacked source archive. Distribution tarballs contain a pre-generated copy
-# of this file.
-
-version_version = '{version}'
-version_full = '{version_full}'
-"""
-        templatefun = r"""
-
-def get_versions(default={}, verbose=False):
-        return {'version': version_version, 'full': version_full}
-"""
-        subst_template = template.format(
+        subst_template = self.template.format(
             version=VERSION_SHORT,
-            version_full=VERSION_FULL) + templatefun
-        with open(versioneer.versionfile_source, 'w') as f:
+            full_revisionid=VERSION_REVISION) + self.templatefun
+        versioneer_cfg = versioneer.get_config_from_root('.')
+        with open(versioneer_cfg.versionfile_source, 'w') as f:
             f.write(subst_template)
 
 
+cmdclass = versioneer.get_cmdclass()
 cmdclass["freeze_debianver"] = freeze_debianver
 
 # XXX add ref to docs
diff --git a/server/src/leap/soledad/server/_version.py b/server/src/leap/soledad/server/_version.py
index 61bb57d9..8c27440f 100644
--- a/server/src/leap/soledad/server/_version.py
+++ b/server/src/leap/soledad/server/_version.py
@@ -1,74 +1,157 @@
+
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
 
-# these strings will be replaced by git during git-archive
+"""Git implementation of _version.py."""
 
+import errno
+import os
+import re
 import subprocess
 import sys
-import re
-import os.path
 
-IN_LONG_VERSION_PY = True
 
-git_refnames = "$Format:%d$"
-git_full = "$Format:%H$"
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "$Format:%d$"
+    git_full = "$Format:%H$"
+    keywords = {"refnames": git_refnames, "full": git_full}
+    return keywords
 
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.cmd on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "pep440"
+    cfg.tag_prefix = ""
+    cfg.parentdir_prefix = "None"
+    cfg.versionfile_source = "src/leap/soledad/server/_version.py"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None
+    else:
         if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
+            print("unable to find command, tried %s" % (commands,))
         return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %s (error)" % args[0])
+            print("unable to run %s (error)" % dispcmd)
         return None
     return stdout
 
 
-def get_expanded_variables(versionfile_source):
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with "
+                  "prefix '%s'" % (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source, "r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {}  # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -84,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
         # "stabilization", as well as "HEAD" and "master".
         tags = set([r for r in refs if re.search(r'\d', r)])
         if verbose:
-            print("discarding '%s', no digits" % ",".join(refs - tags))
+            print("discarding '%s', no digits" % ",".join(refs-tags))
     if verbose:
         print("likely tags: %s" % ",".join(sorted(tags)))
     for ref in sorted(tags):
@@ -94,123 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             if verbose:
                 print("picking %s" % r)
             return {"version": r,
-                    "full": variables["full"].strip()}
-    # no suitable tags, so we use the full revision id
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return {"version": variables["full"].strip(),
-            "full": variables["full"].strip()}
-
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {}  # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(
-            os.path.join('..', here))
 
-    ######################################################
-    # XXX patch for our specific configuration with
-    # the three projects leap.soledad.{common, client, server}
-    # inside the same repo.
-    ######################################################
-    root = os.path.dirname(os.path.join('..', root))
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
 
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
     if not os.path.exists(os.path.join(root, ".git")):
         if verbose:
             print("no .git in %s" % root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.cmd"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" %
-                  (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source,
-                            verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {}  # without __file__, we have no hope
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%s*" % tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
+
+    try:
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
+
+    try:
+        root = os.path.realpath(__file__)
         # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
-    else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree"}
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start "
-                  "with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = ""
-parentdir_prefix = "leap.soledad.server-"
-versionfile_source = "src/leap/soledad/server/_version.py"
-
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = {"refnames": git_refnames, "full": git_full}
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version"}
diff --git a/server/versioneer.py b/server/versioneer.py
index 18dfd923..58339251 100644
--- a/server/versioneer.py
+++ b/server/versioneer.py
@@ -1,170 +1,641 @@
-#! /usr/bin/python
 
-"""versioneer.py
+# Version: 0.16
 
-(like a rocketeer, but for versions)
+"""The Versioneer - like a rocketeer, but for versions.
 
+The Versioneer
+==============
+
+* like a rocketeer, but for versions!
 * https://github.com/warner/python-versioneer
 * Brian Warner
 * License: Public Domain
-* Version: 0.7+
-
-This file helps distutils-based projects manage their version number by just
-creating version-control tags.
-
-For developers who work from a VCS-generated tree (e.g. 'git clone' etc),
-each 'setup.py version', 'setup.py build', 'setup.py sdist' will compute a
-version number by asking your version-control tool about the current
-checkout. The version number will be written into a generated _version.py
-file of your choosing, where it can be included by your __init__.py
-
-For users who work from a VCS-generated tarball (e.g. 'git archive'), it will
-compute a version number by looking at the name of the directory created when
-te tarball is unpacked. This conventionally includes both the name of the
-project and a version number.
-
-For users who work from a tarball built by 'setup.py sdist', it will get a
-version number from a previously-generated _version.py file.
-
-As a result, loading code directly from the source tree will not result in a
-real version. If you want real versions from VCS trees (where you frequently
-update from the upstream repository, or do new development), you will need to
-do a 'setup.py version' after each update, and load code from the build/
-directory.
-
-You need to provide this code with a few configuration values:
-
- versionfile_source:
-    A project-relative pathname into which the generated version strings
-    should be written. This is usually a _version.py next to your project's
-    main __init__.py file. If your project uses src/myproject/__init__.py,
-    this should be 'src/myproject/_version.py'. This file should be checked
-    in to your VCS as usual: the copy created below by 'setup.py
-    update_files' will include code that parses expanded VCS keywords in
-    generated tarballs. The 'build' and 'sdist' commands will replace it with
-    a copy that has just the calculated version string.
-
- versionfile_build:
-    Like versionfile_source, but relative to the build directory instead of
-    the source directory. These will differ when your setup.py uses
-    'package_dir='. If you have package_dir={'myproject': 'src/myproject'},
-    then you will probably have versionfile_build='myproject/_version.py' and
-    versionfile_source='src/myproject/_version.py'.
-
- tag_prefix: a string, like 'PROJECTNAME-', which appears at the start of all
-             VCS tags. If your tags look like 'myproject-1.2.0', then you
-             should use tag_prefix='myproject-'. If you use unprefixed tags
-             like '1.2.0', this should be an empty string.
-
- parentdir_prefix: a string, frequently the same as tag_prefix, which
-                   appears at the start of all unpacked tarball filenames. If
-                   your tarball unpacks into 'myproject-1.2.0', this should
-                   be 'myproject-'.
-
-To use it:
-
- 1: include this file in the top level of your project
- 2: make the following changes to the top of your setup.py:
-     import versioneer
-     versioneer.versionfile_source = 'src/myproject/_version.py'
-     versioneer.versionfile_build = 'myproject/_version.py'
-     versioneer.tag_prefix = '' # tags are like 1.2.0
-     versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0'
- 3: add the following arguments to the setup() call in your setup.py:
-     version=versioneer.get_version(),
-     cmdclass=versioneer.get_cmdclass(),
- 4: run 'setup.py update_files', which will create _version.py, and will
-    modify your __init__.py to define __version__ (by calling a function
-    from _version.py)
- 5: modify your MANIFEST.in to include versioneer.py
- 6: add both versioneer.py and the generated _version.py to your VCS
-"""
+* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy
+* [![Latest Version]
+(https://pypip.in/version/versioneer/badge.svg?style=flat)
+](https://pypi.python.org/pypi/versioneer/)
+* [![Build Status]
+(https://travis-ci.org/warner/python-versioneer.png?branch=master)
+](https://travis-ci.org/warner/python-versioneer)
+
+This is a tool for managing a recorded version number in distutils-based
+python projects. The goal is to remove the tedious and error-prone "update
+the embedded version string" step from your release process. Making a new
+release should be as easy as recording a new tag in your version-control
+system, and maybe making new tarballs.
+
+
+## Quick Install
+
+* `pip install versioneer` to somewhere to your $PATH
+* add a `[versioneer]` section to your setup.cfg (see below)
+* run `versioneer install` in your source tree, commit the results
+
+## Version Identifiers
+
+Source trees come from a variety of places:
+
+* a version-control system checkout (mostly used by developers)
+* a nightly tarball, produced by build automation
+* a snapshot tarball, produced by a web-based VCS browser, like github's
+  "tarball from tag" feature
+* a release tarball, produced by "setup.py sdist", distributed through PyPI
+
+Within each source tree, the version identifier (either a string or a number,
+this tool is format-agnostic) can come from a variety of places:
+
+* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows
+  about recent "tags" and an absolute revision-id
+* the name of the directory into which the tarball was unpacked
+* an expanded VCS keyword ($Id$, etc)
+* a `_version.py` created by some earlier build step
+
+For released software, the version identifier is closely related to a VCS
+tag. Some projects use tag names that include more than just the version
+string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool
+needs to strip the tag prefix to extract the version identifier. For
+unreleased software (between tags), the version identifier should provide
+enough information to help developers recreate the same tree, while also
+giving them an idea of roughly how old the tree is (after version 1.2, before
+version 1.3). Many VCS systems can report a description that captures this,
+for example `git describe --tags --dirty --always` reports things like
+"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the
+0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has
+uncommitted changes.
+
+The version identifier is used for multiple purposes:
+
+* to allow the module to self-identify its version: `myproject.__version__`
+* to choose a name and prefix for a 'setup.py sdist' tarball
+
+## Theory of Operation
+
+Versioneer works by adding a special `_version.py` file into your source
+tree, where your `__init__.py` can import it. This `_version.py` knows how to
+dynamically ask the VCS tool for version information at import time.
+
+`_version.py` also contains `$Revision$` markers, and the installation
+process marks `_version.py` to have this marker rewritten with a tag name
+during the `git archive` command. As a result, generated tarballs will
+contain enough information to get the proper version.
+
+To allow `setup.py` to compute a version too, a `versioneer.py` is added to
+the top level of your source tree, next to `setup.py` and the `setup.cfg`
+that configures it. This overrides several distutils/setuptools commands to
+compute the version when invoked, and changes `setup.py build` and `setup.py
+sdist` to replace `_version.py` with a small static file that contains just
+the generated version data.
+
+## Installation
+
+First, decide on values for the following configuration variables:
+
+* `VCS`: the version control system you use. Currently accepts "git".
+
+* `style`: the style of version string to be produced. See "Styles" below for
+  details. Defaults to "pep440", which looks like
+  `TAG[+DISTANCE.gSHORTHASH[.dirty]]`.
+
+* `versionfile_source`:
+
+  A project-relative pathname into which the generated version strings should
+  be written. This is usually a `_version.py` next to your project's main
+  `__init__.py` file, so it can be imported at runtime. If your project uses
+  `src/myproject/__init__.py`, this should be `src/myproject/_version.py`.
+  This file should be checked in to your VCS as usual: the copy created below
+  by `setup.py setup_versioneer` will include code that parses expanded VCS
+  keywords in generated tarballs. The 'build' and 'sdist' commands will
+  replace it with a copy that has just the calculated version string.
+
+  This must be set even if your project does not have any modules (and will
+  therefore never import `_version.py`), since "setup.py sdist" -based trees
+  still need somewhere to record the pre-calculated version strings. Anywhere
+  in the source tree should do. If there is a `__init__.py` next to your
+  `_version.py`, the `setup.py setup_versioneer` command (described below)
+  will append some `__version__`-setting assignments, if they aren't already
+  present.
+
+* `versionfile_build`:
+
+  Like `versionfile_source`, but relative to the build directory instead of
+  the source directory. These will differ when your setup.py uses
+  'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`,
+  then you will probably have `versionfile_build='myproject/_version.py'` and
+  `versionfile_source='src/myproject/_version.py'`.
+
+  If this is set to None, then `setup.py build` will not attempt to rewrite
+  any `_version.py` in the built tree. If your project does not have any
+  libraries (e.g. if it only builds a script), then you should use
+  `versionfile_build = None`. To actually use the computed version string,
+  your `setup.py` will need to override `distutils.command.build_scripts`
+  with a subclass that explicitly inserts a copy of
+  `versioneer.get_version()` into your script file. See
+  `test/demoapp-script-only/setup.py` for an example.
+
+* `tag_prefix`:
+
+  a string, like 'PROJECTNAME-', which appears at the start of all VCS tags.
+  If your tags look like 'myproject-1.2.0', then you should use
+  tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this
+  should be an empty string, using either `tag_prefix=` or `tag_prefix=''`.
+
+* `parentdir_prefix`:
+
+  a optional string, frequently the same as tag_prefix, which appears at the
+  start of all unpacked tarball filenames. If your tarball unpacks into
+  'myproject-1.2.0', this should be 'myproject-'. To disable this feature,
+  just omit the field from your `setup.cfg`.
+
+This tool provides one script, named `versioneer`. That script has one mode,
+"install", which writes a copy of `versioneer.py` into the current directory
+and runs `versioneer.py setup` to finish the installation.
+
+To versioneer-enable your project:
+
+* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and
+  populating it with the configuration values you decided earlier (note that
+  the option names are not case-sensitive):
+
+  ````
+  [versioneer]
+  VCS = git
+  style = pep440
+  versionfile_source = src/myproject/_version.py
+  versionfile_build = myproject/_version.py
+  tag_prefix =
+  parentdir_prefix = myproject-
+  ````
+
+* 2: Run `versioneer install`. This will do the following:
+
+  * copy `versioneer.py` into the top of your source tree
+  * create `_version.py` in the right place (`versionfile_source`)
+  * modify your `__init__.py` (if one exists next to `_version.py`) to define
+    `__version__` (by calling a function from `_version.py`)
+  * modify your `MANIFEST.in` to include both `versioneer.py` and the
+    generated `_version.py` in sdist tarballs
+
+  `versioneer install` will complain about any problems it finds with your
+  `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all
+  the problems.
+
+* 3: add a `import versioneer` to your setup.py, and add the following
+  arguments to the setup() call:
+
+        version=versioneer.get_version(),
+        cmdclass=versioneer.get_cmdclass(),
+
+* 4: commit these changes to your VCS. To make sure you won't forget,
+  `versioneer install` will mark everything it touched for addition using
+  `git add`. Don't forget to add `setup.py` and `setup.cfg` too.
+
+## Post-Installation Usage
+
+Once established, all uses of your tree from a VCS checkout should get the
+current version string. All generated tarballs should include an embedded
+version string (so users who unpack them will not need a VCS tool installed).
+
+If you distribute your project through PyPI, then the release process should
+boil down to two steps:
+
+* 1: git tag 1.0
+* 2: python setup.py register sdist upload
+
+If you distribute it through github (i.e. users use github to generate
+tarballs with `git archive`), the process is:
+
+* 1: git tag 1.0
+* 2: git push; git push --tags
+
+Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at
+least one tag in its history.
+
+## Version-String Flavors
+
+Code which uses Versioneer can learn about its version string at runtime by
+importing `_version` from your main `__init__.py` file and running the
+`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can
+import the top-level `versioneer.py` and run `get_versions()`.
+
+Both functions return a dictionary with different flavors of version
+information:
+
+* `['version']`: A condensed version string, rendered using the selected
+  style. This is the most commonly used value for the project's version
+  string. The default "pep440" style yields strings like `0.11`,
+  `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section
+  below for alternative styles.
+
+* `['full-revisionid']`: detailed revision identifier. For Git, this is the
+  full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac".
+
+* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that
+  this is only accurate if run in a VCS checkout, otherwise it is likely to
+  be False or None
+
+* `['error']`: if the version string could not be computed, this will be set
+  to a string describing the problem, otherwise it will be None. It may be
+  useful to throw an exception in setup.py if this is set, to avoid e.g.
+  creating tarballs with a version string of "unknown".
+
+Some variants are more useful than others. Including `full-revisionid` in a
+bug report should allow developers to reconstruct the exact code being tested
+(or indicate the presence of local changes that should be shared with the
+developers). `version` is suitable for display in an "about" box or a CLI
+`--version` output: it can be easily compared against release notes and lists
+of bugs fixed in various releases.
+
+The installer adds the following text to your `__init__.py` to place a basic
+version in `YOURPROJECT.__version__`:
+
+    from ._version import get_versions
+    __version__ = get_versions()['version']
+    del get_versions
+
+## Styles
+
+The setup.cfg `style=` configuration controls how the VCS information is
+rendered into a version string.
+
+The default style, "pep440", produces a PEP440-compliant string, equal to the
+un-prefixed tag name for actual releases, and containing an additional "local
+version" section with more detail for in-between builds. For Git, this is
+TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags
+--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the
+tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and
+that this commit is two revisions ("+2") beyond the "0.11" tag. For released
+software (exactly equal to a known tag), the identifier will only contain the
+stripped tag, e.g. "0.11".
+
+Other styles are available. See details.md in the Versioneer source tree for
+descriptions.
+
+## Debugging
+
+Versioneer tries to avoid fatal errors: if something goes wrong, it will tend
+to return a version of "0+unknown". To investigate the problem, run `setup.py
+version`, which will run the version-lookup code in a verbose mode, and will
+display the full contents of `get_versions()` (including the `error` string,
+which may help identify what went wrong).
+
+## Updating Versioneer
+
+To upgrade your project to a new release of Versioneer, do the following:
+
+* install the new Versioneer (`pip install -U versioneer` or equivalent)
+* edit `setup.cfg`, if necessary, to include any new configuration settings
+  indicated by the release notes
+* re-run `versioneer install` in your source tree, to replace
+  `SRC/_version.py`
+* commit any changed files
+
+### Upgrading to 0.16
+
+Nothing special.
+
+### Upgrading to 0.15
+
+Starting with this version, Versioneer is configured with a `[versioneer]`
+section in your `setup.cfg` file. Earlier versions required the `setup.py` to
+set attributes on the `versioneer` module immediately after import. The new
+version will refuse to run (raising an exception during import) until you
+have provided the necessary `setup.cfg` section.
+
+In addition, the Versioneer package provides an executable named
+`versioneer`, and the installation process is driven by running `versioneer
+install`. In 0.14 and earlier, the executable was named
+`versioneer-installer` and was run without an argument.
+
+### Upgrading to 0.14
 
-import os, sys, re
-from distutils.core import Command
-from distutils.command.sdist import sdist as _sdist
-from distutils.command.build import build as _build
+0.14 changes the format of the version string. 0.13 and earlier used
+hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a
+plus-separated "local version" section strings, with dot-separated
+components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old
+format, but should be ok with the new one.
 
-versionfile_source = None
-versionfile_build = None
-tag_prefix = None
-parentdir_prefix = None
+### Upgrading from 0.11 to 0.12
 
-VCS = "git"
-IN_LONG_VERSION_PY = False
+Nothing special.
 
+### Upgrading from 0.10 to 0.11
 
-LONG_VERSION_PY = '''
-IN_LONG_VERSION_PY = True
+You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running
+`setup.py setup_versioneer`. This will enable the use of additional
+version-control systems (SVN, etc) in the future.
+
+## Future Directions
+
+This tool is designed to make it easily extended to other version-control
+systems: all VCS-specific components are in separate directories like
+src/git/ . The top-level `versioneer.py` script is assembled from these
+components by running make-versioneer.py . In the future, make-versioneer.py
+will take a VCS name as an argument, and will construct a version of
+`versioneer.py` that is specific to the given VCS. It might also take the
+configuration arguments that are currently provided manually during
+installation by editing setup.py . Alternatively, it might go the other
+direction and include code from all supported VCS systems, reducing the
+number of intermediate scripts.
+
+
+## License
+
+To make Versioneer easier to embed, all its code is dedicated to the public
+domain. The `_version.py` that it creates is also in the public domain.
+Specifically, both are released under the Creative Commons "Public Domain
+Dedication" license (CC0-1.0), as described in
+https://creativecommons.org/publicdomain/zero/1.0/ .
+
+"""
+
+from __future__ import print_function
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+import errno
+import json
+import os
+import re
+import subprocess
+import sys
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_root():
+    """Get the project root directory.
+
+    We require that all commands are run from the project root, i.e. the
+    directory that contains setup.py, setup.cfg, and versioneer.py .
+    """
+    root = os.path.realpath(os.path.abspath(os.getcwd()))
+    setup_py = os.path.join(root, "setup.py")
+    versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        # allow 'python path/to/setup.py COMMAND'
+        root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
+        setup_py = os.path.join(root, "setup.py")
+        versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        err = ("Versioneer was unable to run the project root directory. "
+               "Versioneer requires setup.py to be executed from "
+               "its immediate directory (like 'python setup.py COMMAND'), "
+               "or in a way that lets it use sys.argv[0] to find the root "
+               "(like 'python path/to/setup.py COMMAND').")
+        raise VersioneerBadRootError(err)
+    try:
+        # Certain runtime workflows (setup.py install/develop in a setuptools
+        # tree) execute all dependencies in a single python process, so
+        # "versioneer" may be imported multiple times, and python's shared
+        # module-import table will cache the first one. So we can't use
+        # os.path.dirname(__file__), as that will find whichever
+        # versioneer.py was first imported, even in later projects.
+        me = os.path.realpath(os.path.abspath(__file__))
+        if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]:
+            print("Warning: build in %s is using versioneer.py from %s"
+                  % (os.path.dirname(me), versioneer_py))
+    except NameError:
+        pass
+    return root
+
+
+def get_config_from_root(root):
+    """Read the project setup.cfg file to determine Versioneer config."""
+    # This might raise EnvironmentError (if setup.cfg is missing), or
+    # configparser.NoSectionError (if it lacks a [versioneer] section), or
+    # configparser.NoOptionError (if it lacks "VCS="). See the docstring at
+    # the top of versioneer.py for instructions on writing your setup.cfg .
+    setup_cfg = os.path.join(root, "setup.cfg")
+    parser = configparser.SafeConfigParser()
+    with open(setup_cfg, "r") as f:
+        parser.readfp(f)
+    VCS = parser.get("versioneer", "VCS")  # mandatory
+
+    def get(parser, name):
+        if parser.has_option("versioneer", name):
+            return parser.get("versioneer", name)
+        return None
+    cfg = VersioneerConfig()
+    cfg.VCS = VCS
+    cfg.style = get(parser, "style") or ""
+    cfg.versionfile_source = get(parser, "versionfile_source")
+    cfg.versionfile_build = get(parser, "versionfile_build")
+    cfg.tag_prefix = get(parser, "tag_prefix")
+    if cfg.tag_prefix in ("''", '""'):
+        cfg.tag_prefix = ""
+    cfg.parentdir_prefix = get(parser, "parentdir_prefix")
+    cfg.verbose = get(parser, "verbose")
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+# these dictionaries contain VCS-specific tools
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None
+    else:
+        if verbose:
+            print("unable to find command, tried %s" % (commands,))
+        return None
+    stdout = p.communicate()[0].strip()
+    if sys.version_info[0] >= 3:
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %s (error)" % dispcmd)
+        return None
+    return stdout
+LONG_VERSION_PY['git'] = '''
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
-
-# these strings will be replaced by git during git-archive
-git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
-git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
 
+"""Git implementation of _version.py."""
 
+import errno
+import os
+import re
 import subprocess
 import sys
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.exe on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
+    git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+    keywords = {"refnames": git_refnames, "full": git_full}
+    return keywords
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "%(STYLE)s"
+    cfg.tag_prefix = "%(TAG_PREFIX)s"
+    cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s"
+    cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %%s" %% dispcmd)
+                print(e)
+            return None
+    else:
         if verbose:
-            print("unable to run %%s" %% args[0])
-            print(e)
+            print("unable to find command, tried %%s" %% (commands,))
         return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %%s (error)" %% args[0])
+            print("unable to run %%s (error)" %% dispcmd)
         return None
     return stdout
 
 
-import sys
-import re
-import os.path
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%%s', but '%%s' doesn't start with "
+                  "prefix '%%s'" %% (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
+
 
-def get_expanded_variables(versionfile_source):
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source,"r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -189,172 +660,350 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %%s" %% r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
-    if not os.path.exists(os.path.join(root, ".git")):
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
+    if not os.path.exists(os.path.join(root, "..", ".git")):
         if verbose:
             print("no .git in %%s" %% root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
-        # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%%s*" %% tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%%s'"
+                               %% describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%%s' doesn't start with prefix '%%s'"
+                print(fmt %% (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'"
+                               %% (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
     else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+        # exception #1
+        rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%%d" %% pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%%d" %% pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%%s" %% pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%%s" %% pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %%
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = "%(TAG_PREFIX)s"
-parentdir_prefix = "%(PARENTDIR_PREFIX)s"
-versionfile_source = "%(VERSIONFILE_SOURCE)s"
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = { "refnames": git_refnames, "full": git_full }
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
 
-'''
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
 
+    Like 'git describe --tags --dirty --always'.
 
-import subprocess
-import sys
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%%s'" %% style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
 
-def run_command(args, cwd=None, verbose=False):
     try:
-        # remember shell=False, so use git.exe on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
-        if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
-        return None
-    stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
-        stdout = stdout.decode()
-    if p.returncode != 0:
-        if verbose:
-            print("unable to run %s (error)" % args[0])
-        return None
-    return stdout
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
 
+    try:
+        root = os.path.realpath(__file__)
+        # versionfile_source is the relative path from the top of the source
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
+            root = os.path.dirname(root)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree"}
 
-import sys
-import re
-import os.path
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
 
-def get_expanded_variables(versionfile_source):
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version"}
+'''
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source,"r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -379,117 +1028,122 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %s" % r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = os.path.dirname(
-        os.path.join('..', here))
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(
-            os.path.join('..', here))
 
-    ######################################################
-    # XXX patch for our specific configuration with
-    # the three projects leap.soledad.{common, client, server}
-    # inside the same repo.
-    ######################################################
-    root = os.path.dirname(os.path.join('..', root))
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
 
-    if not os.path.exists(os.path.join(root, ".git")):
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
+    if not os.path.exists(os.path.join(root, "..", ".git")):
         if verbose:
             print("no .git in %s" % root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(os.path.join('..', __file__))
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
-        # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%s*" % tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
     else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
+    return pieces
 
-import sys
 
-def do_vcs_install(versionfile_source, ipy):
-    GIT = "git"
+def do_vcs_install(manifest_in, versionfile_source, ipy):
+    """Git-specific installation logic for Versioneer.
+
+    For Git, this means creating/changing .gitattributes to mark _version.py
+    for export-time keyword substitution.
+    """
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    run_command([GIT, "add", "versioneer.py"])
-    run_command([GIT, "add", versionfile_source])
-    run_command([GIT, "add", ipy])
+        GITS = ["git.cmd", "git.exe"]
+    files = [manifest_in, versionfile_source]
+    if ipy:
+        files.append(ipy)
+    try:
+        me = __file__
+        if me.endswith(".pyc") or me.endswith(".pyo"):
+            me = os.path.splitext(me)[0] + ".py"
+        versioneer_file = os.path.relpath(me)
+    except NameError:
+        versioneer_file = "versioneer.py"
+    files.append(versioneer_file)
     present = False
     try:
         f = open(".gitattributes", "r")
@@ -504,135 +1158,487 @@ def do_vcs_install(versionfile_source, ipy):
         f = open(".gitattributes", "a+")
         f.write("%s export-subst\n" % versionfile_source)
         f.close()
-        run_command([GIT, "add", ".gitattributes"])
+        files.append(".gitattributes")
+    run_command(GITS, ["add", "--"] + files)
+
 
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with "
+                  "prefix '%s'" % (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
 
 SHORT_VERSION_PY = """
-# This file was generated by 'versioneer.py' (0.7+) from
+# This file was generated by 'versioneer.py' (0.16) from
 # revision-control system data, or from the parent directory name of an
 # unpacked source archive. Distribution tarballs contain a pre-generated copy
 # of this file.
 
-version_version = '%(version)s'
-version_full = '%(full)s'
-def get_versions(default={}, verbose=False):
-    return {'version': version_version, 'full': version_full}
+import json
+import sys
+
+version_json = '''
+%s
+'''  # END VERSION_JSON
 
+
+def get_versions():
+    return json.loads(version_json)
 """
 
-DEFAULT = {"version": "unknown", "full": "unknown"}
 
 def versions_from_file(filename):
-    versions = {}
+    """Try to determine the version from _version.py if present."""
     try:
-        f = open(filename)
+        with open(filename) as f:
+            contents = f.read()
     except EnvironmentError:
-        return versions
-    for line in f.readlines():
-        mo = re.match("version_version = '([^']+)'", line)
-        if mo:
-            versions["version"] = mo.group(1)
-        mo = re.match("version_full = '([^']+)'", line)
-        if mo:
-            versions["full"] = mo.group(1)
-    f.close()
-    return versions
+        raise NotThisMethod("unable to read _version.py")
+    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
+                   contents, re.M | re.S)
+    if not mo:
+        raise NotThisMethod("no version_json in _version.py")
+    return json.loads(mo.group(1))
+
 
 def write_to_version_file(filename, versions):
-    f = open(filename, "w")
-    f.write(SHORT_VERSION_PY % versions)
-    f.close()
+    """Write the given version number to the given _version.py file."""
+    os.unlink(filename)
+    contents = json.dumps(versions, sort_keys=True,
+                          indent=1, separators=(",", ": "))
+    with open(filename, "w") as f:
+        f.write(SHORT_VERSION_PY % contents)
+
     print("set %s to '%s'" % (filename, versions["version"]))
 
 
-def get_best_versions(versionfile, tag_prefix, parentdir_prefix,
-                      default=DEFAULT, verbose=False):
-    # returns dict with two keys: 'version' and 'full'
-    #
-    # extract version from first of _version.py, 'git describe', parentdir.
-    # This is meant to work for developers using a source checkout, for users
-    # of a tarball created by 'setup.py sdist', and for users of a
-    # tarball/zipball created by 'git archive' or github's download-from-tag
-    # feature.
-
-    variables = get_expanded_variables(versionfile_source)
-    if variables:
-        ver = versions_from_expanded_variables(variables, tag_prefix)
-        if ver:
-            if verbose: print("got version from expanded variable %s" % ver)
-            return ver
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
 
-    ver = versions_from_file(versionfile)
-    if ver:
-        if verbose: print("got version from file %s %s" % (versionfile, ver))
-        return ver
 
-    ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if ver:
-        if verbose: print("got version from git %s" % ver)
-        return ver
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
 
-    ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose)
-    if ver:
-        if verbose: print("got version from parentdir %s" % ver)
-        return ver
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
 
-    if verbose: print("got version from default %s" % ver)
-    return default
-
-def get_versions(default=DEFAULT, verbose=False):
-    assert versionfile_source is not None, "please set versioneer.versionfile_source"
-    assert tag_prefix is not None, "please set versioneer.tag_prefix"
-    assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix"
-    return get_best_versions(versionfile_source, tag_prefix, parentdir_prefix,
-                             default=default, verbose=verbose)
-def get_version(verbose=False):
-    return get_versions(verbose=verbose)["version"]
-
-class cmd_version(Command):
-    description = "report generated version string"
-    user_options = []
-    boolean_options = []
-    def initialize_options(self):
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+class VersioneerBadRootError(Exception):
+    """The project root directory is unknown or missing key files."""
+
+
+def get_versions(verbose=False):
+    """Get the project version from whatever source is available.
+
+    Returns dict with two keys: 'version' and 'full'.
+    """
+    if "versioneer" in sys.modules:
+        # see the discussion in cmdclass.py:get_cmdclass()
+        del sys.modules["versioneer"]
+
+    root = get_root()
+    cfg = get_config_from_root(root)
+
+    assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg"
+    handlers = HANDLERS.get(cfg.VCS)
+    assert handlers, "unrecognized VCS '%s'" % cfg.VCS
+    verbose = verbose or cfg.verbose
+    assert cfg.versionfile_source is not None, \
+        "please set versioneer.versionfile_source"
+    assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix"
+
+    versionfile_abs = os.path.join(root, cfg.versionfile_source)
+
+    # extract version from first of: _version.py, VCS command (e.g. 'git
+    # describe'), parentdir. This is meant to work for developers using a
+    # source checkout, for users of a tarball created by 'setup.py sdist',
+    # and for users of a tarball/zipball created by 'git archive' or github's
+    # download-from-tag feature or the equivalent in other VCSes.
+
+    get_keywords_f = handlers.get("get_keywords")
+    from_keywords_f = handlers.get("keywords")
+    if get_keywords_f and from_keywords_f:
+        try:
+            keywords = get_keywords_f(versionfile_abs)
+            ver = from_keywords_f(keywords, cfg.tag_prefix, verbose)
+            if verbose:
+                print("got version from expanded keyword %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        ver = versions_from_file(versionfile_abs)
+        if verbose:
+            print("got version from file %s %s" % (versionfile_abs, ver))
+        return ver
+    except NotThisMethod:
         pass
-    def finalize_options(self):
+
+    from_vcs_f = handlers.get("pieces_from_vcs")
+    if from_vcs_f:
+        try:
+            pieces = from_vcs_f(cfg.tag_prefix, root, verbose)
+            ver = render(pieces, cfg.style)
+            if verbose:
+                print("got version from VCS %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        if cfg.parentdir_prefix:
+            ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+            if verbose:
+                print("got version from parentdir %s" % ver)
+            return ver
+    except NotThisMethod:
         pass
-    def run(self):
-        ver = get_version(verbose=True)
-        print("Version is currently: %s" % ver)
-
-
-class cmd_build(_build):
-    def run(self):
-        versions = get_versions(verbose=True)
-        _build.run(self)
-        # now locate _version.py in the new build/ directory and replace it
-        # with an updated value
-        target_versionfile = os.path.join(self.build_lib, versionfile_build)
-        print("UPDATING %s" % target_versionfile)
-        os.unlink(target_versionfile)
-        f = open(target_versionfile, "w")
-        f.write(SHORT_VERSION_PY % versions)
-        f.close()
 
-class cmd_sdist(_sdist):
-    def run(self):
-        versions = get_versions(verbose=True)
-        self._versioneer_generated_versions = versions
-        # unless we update this, the command will keep using the old version
-        self.distribution.metadata.version = versions["version"]
-        return _sdist.run(self)
-
-    def make_release_tree(self, base_dir, files):
-        _sdist.make_release_tree(self, base_dir, files)
-        # now locate _version.py in the new base_dir directory (remembering
-        # that it may be a hardlink) and replace it with an updated value
-        target_versionfile = os.path.join(base_dir, versionfile_source)
-        print("UPDATING %s" % target_versionfile)
-        os.unlink(target_versionfile)
-        f = open(target_versionfile, "w")
-        f.write(SHORT_VERSION_PY % self._versioneer_generated_versions)
-        f.close()
+    if verbose:
+        print("unable to compute version")
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None, "error": "unable to compute version"}
+
+
+def get_version():
+    """Get the short version string for this project."""
+    return get_versions()["version"]
+
+
+def get_cmdclass():
+    """Get the custom setuptools/distutils subclasses used by Versioneer."""
+    if "versioneer" in sys.modules:
+        del sys.modules["versioneer"]
+        # this fixes the "python setup.py develop" case (also 'install' and
+        # 'easy_install .'), in which subdependencies of the main project are
+        # built (using setup.py bdist_egg) in the same python process. Assume
+        # a main project A and a dependency B, which use different versions
+        # of Versioneer. A's setup.py imports A's Versioneer, leaving it in
+        # sys.modules by the time B's setup.py is executed, causing B to run
+        # with the wrong versioneer. Setuptools wraps the sub-dep builds in a
+        # sandbox that restores sys.modules to it's pre-build state, so the
+        # parent is protected against the child's "import versioneer". By
+        # removing ourselves from sys.modules here, before the child build
+        # happens, we protect the child from the parent's versioneer too.
+        # Also see https://github.com/warner/python-versioneer/issues/52
+
+    cmds = {}
+
+    # we add "version" to both distutils and setuptools
+    from distutils.core import Command
+
+    class cmd_version(Command):
+        description = "report generated version string"
+        user_options = []
+        boolean_options = []
+
+        def initialize_options(self):
+            pass
+
+        def finalize_options(self):
+            pass
+
+        def run(self):
+            vers = get_versions(verbose=True)
+            print("Version: %s" % vers["version"])
+            print(" full-revisionid: %s" % vers.get("full-revisionid"))
+            print(" dirty: %s" % vers.get("dirty"))
+            if vers["error"]:
+                print(" error: %s" % vers["error"])
+    cmds["version"] = cmd_version
+
+    # we override "build_py" in both distutils and setuptools
+    #
+    # most invocation pathways end up running build_py:
+    #  distutils/build -> build_py
+    #  distutils/install -> distutils/build ->..
+    #  setuptools/bdist_wheel -> distutils/install ->..
+    #  setuptools/bdist_egg -> distutils/install_lib -> build_py
+    #  setuptools/install -> bdist_egg ->..
+    #  setuptools/develop -> ?
+
+    # we override different "build_py" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.build_py import build_py as _build_py
+    else:
+        from distutils.command.build_py import build_py as _build_py
+
+    class cmd_build_py(_build_py):
+        def run(self):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            versions = get_versions()
+            _build_py.run(self)
+            # now locate _version.py in the new build/ directory and replace
+            # it with an updated value
+            if cfg.versionfile_build:
+                target_versionfile = os.path.join(self.build_lib,
+                                                  cfg.versionfile_build)
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+    cmds["build_py"] = cmd_build_py
+
+    if "cx_Freeze" in sys.modules:  # cx_freeze enabled?
+        from cx_Freeze.dist import build_exe as _build_exe
+
+        class cmd_build_exe(_build_exe):
+            def run(self):
+                root = get_root()
+                cfg = get_config_from_root(root)
+                versions = get_versions()
+                target_versionfile = cfg.versionfile_source
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+
+                _build_exe.run(self)
+                os.unlink(target_versionfile)
+                with open(cfg.versionfile_source, "w") as f:
+                    LONG = LONG_VERSION_PY[cfg.VCS]
+                    f.write(LONG %
+                            {"DOLLAR": "$",
+                             "STYLE": cfg.style,
+                             "TAG_PREFIX": cfg.tag_prefix,
+                             "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                             "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                             })
+        cmds["build_exe"] = cmd_build_exe
+        del cmds["build_py"]
+
+    # we override different "sdist" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.sdist import sdist as _sdist
+    else:
+        from distutils.command.sdist import sdist as _sdist
+
+    class cmd_sdist(_sdist):
+        def run(self):
+            versions = get_versions()
+            self._versioneer_generated_versions = versions
+            # unless we update this, the command will keep using the old
+            # version
+            self.distribution.metadata.version = versions["version"]
+            return _sdist.run(self)
+
+        def make_release_tree(self, base_dir, files):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            _sdist.make_release_tree(self, base_dir, files)
+            # now locate _version.py in the new base_dir directory
+            # (remembering that it may be a hardlink) and replace it with an
+            # updated value
+            target_versionfile = os.path.join(base_dir, cfg.versionfile_source)
+            print("UPDATING %s" % target_versionfile)
+            write_to_version_file(target_versionfile,
+                                  self._versioneer_generated_versions)
+    cmds["sdist"] = cmd_sdist
+
+    return cmds
+
+
+CONFIG_ERROR = """
+setup.cfg is missing the necessary Versioneer configuration. You need
+a section like:
+
+ [versioneer]
+ VCS = git
+ style = pep440
+ versionfile_source = src/myproject/_version.py
+ versionfile_build = myproject/_version.py
+ tag_prefix =
+ parentdir_prefix = myproject-
+
+You will also need to edit your setup.py to use the results:
+
+ import versioneer
+ setup(version=versioneer.get_version(),
+       cmdclass=versioneer.get_cmdclass(), ...)
+
+Please read the docstring in ./versioneer.py for configuration instructions,
+edit setup.cfg, and re-run the installer or 'python versioneer.py setup'.
+"""
+
+SAMPLE_CONFIG = """
+# See the docstring in versioneer.py for instructions. Note that you must
+# re-run 'versioneer.py setup' after changing this section, and commit the
+# resulting files.
+
+[versioneer]
+#VCS = git
+#style = pep440
+#versionfile_source =
+#versionfile_build =
+#tag_prefix =
+#parentdir_prefix =
+
+"""
 
 INIT_PY_SNIPPET = """
 from ._version import get_versions
@@ -640,40 +1646,129 @@ __version__ = get_versions()['version']
 del get_versions
 """
 
-class cmd_update_files(Command):
-    description = "modify __init__.py and create _version.py"
-    user_options = []
-    boolean_options = []
-    def initialize_options(self):
-        pass
-    def finalize_options(self):
-        pass
-    def run(self):
-        ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py")
-        print(" creating %s" % versionfile_source)
-        f = open(versionfile_source, "w")
-        f.write(LONG_VERSION_PY % {"DOLLAR": "$",
-                                   "TAG_PREFIX": tag_prefix,
-                                   "PARENTDIR_PREFIX": parentdir_prefix,
-                                   "VERSIONFILE_SOURCE": versionfile_source,
-                                   })
-        f.close()
+
+def do_setup():
+    """Main VCS-independent setup function for installing Versioneer."""
+    root = get_root()
+    try:
+        cfg = get_config_from_root(root)
+    except (EnvironmentError, configparser.NoSectionError,
+            configparser.NoOptionError) as e:
+        if isinstance(e, (EnvironmentError, configparser.NoSectionError)):
+            print("Adding sample versioneer config to setup.cfg",
+                  file=sys.stderr)
+            with open(os.path.join(root, "setup.cfg"), "a") as f:
+                f.write(SAMPLE_CONFIG)
+        print(CONFIG_ERROR, file=sys.stderr)
+        return 1
+
+    print(" creating %s" % cfg.versionfile_source)
+    with open(cfg.versionfile_source, "w") as f:
+        LONG = LONG_VERSION_PY[cfg.VCS]
+        f.write(LONG % {"DOLLAR": "$",
+                        "STYLE": cfg.style,
+                        "TAG_PREFIX": cfg.tag_prefix,
+                        "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                        "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                        })
+
+    ipy = os.path.join(os.path.dirname(cfg.versionfile_source),
+                       "__init__.py")
+    if os.path.exists(ipy):
         try:
-            old = open(ipy, "r").read()
+            with open(ipy, "r") as f:
+                old = f.read()
         except EnvironmentError:
             old = ""
         if INIT_PY_SNIPPET not in old:
             print(" appending to %s" % ipy)
-            f = open(ipy, "a")
-            f.write(INIT_PY_SNIPPET)
-            f.close()
+            with open(ipy, "a") as f:
+                f.write(INIT_PY_SNIPPET)
         else:
             print(" %s unmodified" % ipy)
-        do_vcs_install(versionfile_source, ipy)
+    else:
+        print(" %s doesn't exist, ok" % ipy)
+        ipy = None
+
+    # Make sure both the top-level "versioneer.py" and versionfile_source
+    # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so
+    # they'll be copied into source distributions. Pip won't be able to
+    # install the package without this.
+    manifest_in = os.path.join(root, "MANIFEST.in")
+    simple_includes = set()
+    try:
+        with open(manifest_in, "r") as f:
+            for line in f:
+                if line.startswith("include "):
+                    for include in line.split()[1:]:
+                        simple_includes.add(include)
+    except EnvironmentError:
+        pass
+    # That doesn't cover everything MANIFEST.in can do
+    # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
+    # it might give some false negatives. Appending redundant 'include'
+    # lines is safe, though.
+    if "versioneer.py" not in simple_includes:
+        print(" appending 'versioneer.py' to MANIFEST.in")
+        with open(manifest_in, "a") as f:
+            f.write("include versioneer.py\n")
+    else:
+        print(" 'versioneer.py' already in MANIFEST.in")
+    if cfg.versionfile_source not in simple_includes:
+        print(" appending versionfile_source ('%s') to MANIFEST.in" %
+              cfg.versionfile_source)
+        with open(manifest_in, "a") as f:
+            f.write("include %s\n" % cfg.versionfile_source)
+    else:
+        print(" versionfile_source already in MANIFEST.in")
 
-def get_cmdclass():
-    return {'version': cmd_version,
-            'update_files': cmd_update_files,
-            'build': cmd_build,
-            'sdist': cmd_sdist,
-            }
+    # Make VCS-specific changes. For git, this means creating/changing
+    # .gitattributes to mark _version.py for export-time keyword
+    # substitution.
+    do_vcs_install(manifest_in, cfg.versionfile_source, ipy)
+    return 0
+
+
+def scan_setup_py():
+    """Validate the contents of setup.py against Versioneer's expectations."""
+    found = set()
+    setters = False
+    errors = 0
+    with open("setup.py", "r") as f:
+        for line in f.readlines():
+            if "import versioneer" in line:
+                found.add("import")
+            if "versioneer.get_cmdclass()" in line:
+                found.add("cmdclass")
+            if "versioneer.get_version()" in line:
+                found.add("get_version")
+            if "versioneer.VCS" in line:
+                setters = True
+            if "versioneer.versionfile_source" in line:
+                setters = True
+    if len(found) != 3:
+        print("")
+        print("Your setup.py appears to be missing some important items")
+        print("(but I might be wrong). Please make sure it has something")
+        print("roughly like the following:")
+        print("")
+        print(" import versioneer")
+        print(" setup( version=versioneer.get_version(),")
+        print("        cmdclass=versioneer.get_cmdclass(),  ...)")
+        print("")
+        errors += 1
+    if setters:
+        print("You should remove lines like 'versioneer.VCS = ' and")
+        print("'versioneer.versionfile_source = ' . This configuration")
+        print("now lives in setup.cfg, and should be removed from setup.py")
+        print("")
+        errors += 1
+    return errors
+
+if __name__ == "__main__":
+    cmd = sys.argv[1]
+    if cmd == "setup":
+        errors = do_setup()
+        errors += scan_setup_py()
+        if errors:
+            sys.exit(1)
-- 
cgit v1.2.3


From d5974f61df6ca640dde69903976d4721b6d88007 Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Mon, 18 Apr 2016 11:29:46 -0400
Subject: [pkg] Update changelog

---
 CHANGELOG                                          | 378 ---------------------
 CHANGELOG.rst                                      |  55 +++
 HISTORY                                            | 378 +++++++++++++++++++++
 .../changes/bug_7503-do-not-signal-sync-complete   |   1 -
 client/changes/bug_missing_design_doc_handler      |   1 -
 client/changes/feat_send_batch                     |   1 -
 client/changes/feat_use_cryptography               |   1 -
 client/changes/next-changelog.rst                  |   6 +-
 common/changes/create_db_cmd                       |   2 -
 common/changes/next-changelog.rst                  |   3 +-
 server/changes/change_soledad_configdir            |   2 -
 server/changes/create_db_cmd                       |   4 -
 server/changes/feat_configurable_ensure            |   5 -
 server/changes/feat_handle_send_batch_better       |   1 -
 server/changes/next-changelog.rst                  |  28 ++
 15 files changed, 464 insertions(+), 402 deletions(-)
 delete mode 100644 CHANGELOG
 create mode 100644 CHANGELOG.rst
 create mode 100644 HISTORY
 delete mode 100644 client/changes/bug_7503-do-not-signal-sync-complete
 delete mode 100644 client/changes/bug_missing_design_doc_handler
 delete mode 100644 client/changes/feat_send_batch
 delete mode 100644 client/changes/feat_use_cryptography
 delete mode 100644 common/changes/create_db_cmd
 delete mode 100644 server/changes/change_soledad_configdir
 delete mode 100644 server/changes/create_db_cmd
 delete mode 100644 server/changes/feat_configurable_ensure
 delete mode 100644 server/changes/feat_handle_send_batch_better
 create mode 100644 server/changes/next-changelog.rst

diff --git a/CHANGELOG b/CHANGELOG
deleted file mode 100644
index b0530226..00000000
--- a/CHANGELOG
+++ /dev/null
@@ -1,378 +0,0 @@
-0.7.4 Oct 28, 2015:
-Client:
-  o Do not signal sync completion if sync failed. Closes: #7503
-  o Handle missing design doc at GET (get_sync_info). Soledad server can handle
-    this during sync.
-  o Increase http request timeout time to 90s. Related: #7382.
-  o Handle DatabaseDoesNotExist during sync.
-  o Handle MissingDesignDocError after get_sync_info.
-
-0.7.3 Sep 22, 2015:
-Client:
-  o Bugfix: refactor code loss. Closes #7412.
-  o Bugfix: Set active secret before saving local file.
-  o Split http_target into 4 modules, separating those responsibilities.
-  o Refactor details of making an HTTP request body and headers out of the
-    send/fetch logic. This also makes it easier to enable batching.
-
-Server:
-  o Fix a bug where BadRequest could be raised after everything was persisted.
-
-Common:
-  o Refactor couch.py to separate persistence from logic while saving uploaded
-    documents. Also simplify logic while checking for conflicts.
-
-
-0.7.2 Aug 26, 2015:
-Client:
-  o Remove MAC from secrets file. Closes #6980.
-  o refactor SoledadCrypto to remove circular dependency with SoledadSecrets.
-    Closes #7338.
-  o Improve how we send information on SOLEDAD_SYNC_SEND_STATUS and in
-    SOLEDAD_SYNC_RECEIVE_STATUS. Related to Feature #7353.
-  o Fix hanging sync by properly waiting db initialization on sync decrypter
-    pool. Closes #7386.
-  o Avoid double decryption of documents.
-  o Fix the order of the events emited for incoming documents.
-  o bugfix: move sync db and encpool creation to api.
-  o Bugfix: wrong sqlcipher passphrase now raises correctly.
-  o Refactor decription pool and http target to use a deferred instead of a
-    waiting loop.
-  o Improve helper scripts and dependencies listing.
-  o Refactor enc/dec pool to standardize start/stop of the pools.
-  o Add tests for enc/dec pool.
-
-Client, Server, Common:
-  o Lots of code restyling to pass CI tests.
-  o Lots of work done to get tests passing.
-  o Remove dependency on simplejson.
-
-0.7.1 Jul 10, 2015:
-Client:
-  o Fix sync encrypter pool close queue error. Closes #7088.
-  o Expose post-sync hooks via plugin system. Related: #6996.
-
-0.7.0 Jun 01 2015:
-Client:
-  o Do not depend on pysqlite2 (#2945).
-  o Reset syncer connection when getting HTTP error during sync (#5855).
-  o Wait for last post request to finish before starting a new one (#5975).
-  o Use TLS v1 in soledad client. Fixes partially #6437
-  o Retry on sqlcipher thread timeouts (#6625).
-  o Fix the order of insertion of documents when using workers for decrypting
-    incoming documents during a sync. Closes #6757.
-  o Add dependency on Twisted. Closes #6797.
-  o Fix the log message when a local secret is not found so it's less
-    confusing. Closes #6892.
-  o Always initialize the sync db to allow for both asynchronous encryption
-    and asynchronous decryption when syncing.
-  o Fallback to utf-8 if confidence on chardet guessing is too low.
-  o Refactor asynchronous encryption/decryption code to its own file.
-  o Fix logging and graceful failing when exceptions are raised during sync.
-  o Improve log messages when concurrently fetching documents from the server.
-  o Store all incoming documents in the sync db (#5895).
-  o Include the IV in the encrypted document MAC (#6400).
-  o Adapt soledad to the new events api on leap.common. Related to #6359.
-  o Add a pool of HTTP/HTTPS connections that is able to verify the server
-    certificate against a given CA certificate.
-  o Use twisted.enterprise.adbapi for access to the sync database.
-  o Use twisted.web.client for client sync.
-
-Common:
-  o Include couch design docs source files in source distribution and only
-    compile ddocs.py when building the package (#5896).
-  o Bail out if cdocs/ dir does not exist. Closes: #6671
-  o Remove unneeded parameters from CouchServerState initialization. Closes
-    #6833.
-  o  Adapt soledad to the new events api on leap.common. Related to #6359.
-
-Server:
-  o Run daemon as user soledad (#6436).
-  o Avoid use of SSLv3 (#6437).
-  o Fix server initscript location (#6557).
-  o Add dependency on Twisted. Closes #6797.
-  o Remove unneeded parameters from CouchServerState initialization. Closes
-    #6833.
-  o Fix server daemon uid and gid by passing them to twistd on the initscript.
-  o Use monthly token databases. Closes #6785.
-
-0.6.5 Apr 09 2015:
-Server:
-  o Remove unneeded parameters from CouchServerState initialization. Closes
-    #6833.
-
-0.6.4 Jan 23 2015:
-Common:
-  o Bail out if cdocs/ dir does not exist. Closes: #6671
-
-0.6.3 Dec 16 2014:
-Client:
-  o Fix deferred encryption/decryption parameters (#6500).
-
-0.6.2 Dec 11 2014:
-Client:
-  o Fix incorrect ssl context setup.
-
-0.6.1 Dec 08 2014:
-Client:
-  o Use TLS v1 in soledad client. Fixes partially #6437
-
-Server:
-  o Run daemon as user soledad (#6436).
-  o Avoid use of SSLv3 (#6437).
-
-0.6.0 Jul 18, 2014:
-Client:
-  o Close all connections after syncing. Fixes #5518.
-  o Reset synchronizer state in order to reuse the same synchronizer
-    object multiple times.
-  o Use temporal database for encryption/decryption during
-    sync. Closes #5326.
-  o Add sync status signals. Closes #5517.
-  o Allow for interrupting and recovering sync. Closes #5517.
-  o Parallelize sync requests and reuse HTTP connections.
-  o Split sync in multiple POST requests in client. Closes #5571.
-
-Common:
-  o Use a dedicated HTTP resource for couch multipart PUTs to avoid
-    bigcouch. Closes #5739.
-
-Server:
-  o Pin PyOpenSSL dependency version to <0.14 to avoid yet another
-    crypto dependency.
-  o Authenticate in time-insensitive manner. Closes #3399.
-  o Allow for interrupting and recovering sync. Closes #5517.
-  o Split sync in multiple POST requests in server. Closes #5571.
-
-0.5.2 Jun 6, 2014:
-Client:
-  o Reset synchronizer state in order to reuse the same synchronizer
-    object multiple times.
-  o Add sync status signals. Closes #5517.
-  o Allow for interrupting and recovering sync. Closes #5517.
-  o Split sync in multiple POST requests in client. Closes #5571.
-
-Common:
-  o Use a dedicated HTTP resource for couch multipart PUTs to avoid bigcouch
-    bug. Fixes #5739.
-
-Server:
-  o Allow for interrupting and recovering sync. Closes #5517.
-  o Split sync in multiple POST requests in server. Closes #5571.
-  o Authenticate in time-insensitive manner. Closes #3399.
-
-0.5.1 May 16, 2014:
-Client:
-  o Close connection with server after syncing to avoid client hanging
-    on exit. Fixes #5507.
-
-Common:
-  o Properly close connections on couch backend. Also prevent file
-    descriptor leaks on tests. Closes #5493.
-  o Prevent couch backend from always trying to create the
-    database. Fixes #5386.
-  o Prevent Couch Server State from making one uneeded GET request on
-    instantiation. Fixes #5386.
-
-0.5.0 Apr 4, 2014:
-Client:
-  o Catch lock timeout exception. Fixes #4435.
-  o Add lock for create_doc and update_indexes call, prevents
-    concurrent access to the db. Closes #5139.
-  o Back-compatibility for socket.create_connection interface in
-    2.6. Closes #5208.
-  o Always return unicode in helper method, even on
-    UnicodeError. Related to #4998.
-  o Fix a bug in soledad.client.sqlcipher by which we were creating a
-    new connection for each sync.
-  o Unlock shared_db if anything fails in the bootstrap
-    sequence. Fixes #4702.
-  o Avoid concurrent syncs for the same account, but allow for
-    distinct accounts. Fixes #4451.
-  o Adds a get_count_by_index to sqlcipher u1db backend. Related to:
-    #4616.
-  o Do not autocreate remote user database when syncing. Tapicero
-    should make sure that that db is created when the user is
-    created. Closes #5302.
-  o Add a read-write lock for all client operations. Addresses: #4972
-  o Add sync=off and tem_store=mem to soledad client, for
-    optimization.
-
-Common:
-  o Add lock timeout HTTP error. Fixes #4435.
-  o Remodel couch backend to fix concurrency and scalability. Closes
-    #4475, #4682, #4683 and #4680.
-  o Remove check for design docs on couch server state initialization
-    Closes #5387.
-  o Renew HTTP session after multipart PUTs to avoid request hanging.
-    Fixes #5449.
-  o Preload time.strptime() to avoid multi-threaded problem on couch
-    backend get_docs() method. Fixes #5449.
-  o Improve error messages. Closes #5035.
-  o Add MissingTokenError and InvalidTokenError as sub exceptions
-    from Unauthorized.
-  o Allow sync of large files (~100MB). Closes #4836.
-  o Add exceptions to deal with missing design documents. Fixes #4994.
-  o Parallelize get_docs() on couch backend to accelerate sync.
-    Closes #5008.
-  o Use less memory when putting docs on couch. Fixes #5011.
-  o Prevent CouchServerState from creating or deleting databases. This
-    way, Soledad remote clients won't ever be able to do these
-    operations when syncing. Part of #5302.
-  o Avoid concurrent syncs problem by adding a lock for PUTting to the
-    sync log update handler. Fixes #5388.
-  o Remove check for couch permissions when CouchServerState is
-    instantiated. This is not necessary anymore because platform
-    takes care of giving the soledad user enough permissions and
-    tapicero takes care of uploading the needed design documents.
-
-Server:
-  o Send propper lock timeout response. Fixes #4435.
-  o Fix raising of auth token errors. Fixes #5191.
-  o Allow sync of large files (~100MB). Closes #4836.
-  o Use a temporary directory for server side locks. Fixes #4918.
-  o Catch couchdb.http.ResourceNotFound exceptions when accessing
-    design documents on couch backend, and raise appropriate missing
-    design documents exceptions. Fixes #4994.
-  o Do not try to create the shared database when running the Soledad
-    Server application. Fixes #5302.
-  o Enable Gzip compression on the soledad wsgi app.
-
-0.4.4 Dec 6, 2013:
-Client:
-  o Add MAC verirication to the recovery document and
-    soledad.json. Closes #4348.
-Common:
-  o Add unicode conversion to put_doc(). Closes #4095.
-  o Remove tests dependency on nose2. Closes #4258.
-
-0.4.3 Nov 15, 2013:
-Client:
-  o Defaults detected encoding to utf-8 to avoid bug if detected
-    encoding is None. Closes: #4417
-  o Open db in autocommit mode, to avoid nested transactions problems.
-    Closes: #4400
-
-0.4.2 Nov 1, 2013:
-Client:
-  o Support non-ascii passwords. Closes #4001.
-  o Change error severity for missing secrets path.
-  o Use chardet as fallback if cchardet not found.
-  o Improve bootstrap sequence and allow for locking the shared
-    database while creating/uploading the encryption secret. Closes
-    #4097.
-Common:
-  o Move some common functions and global variables to
-    leap.soledad.common.
-Server:
-  o Allow for locking the shared database. Closes #4097.
-
-0.4.1 Oct 4, 2013:
-Client:
-  o Save only UTF8 strings. Related to #3660.
-
-0.4.0 Sep 20, 2013:
-Client:
-  o Remove redundant logging when creating data dirs.
-Server:
-  o Verify for couch permissions when starting server. Closes #3501.
-Common:
-  o Improve u1db data storage in couch. Closes #3647.
-  o Turn couchdb dependency for common into optional. Closes #2167.
-  o Add verification for couch permissions. Closes #3501.
-
-0.3.2 Sep 6, 2013:
-Client:
-  o Use dirspec instead of plain xdg. Closes #3574.
-Server:
-  o Fix the init script for soledad server so that it uses the new
-    package namespace.
-
-0.3.1 Aug 23, 2013:
-Client:
-  o Add libsqlite3-dev requirement for soledad.
-  o Check for None in private methods that depend on _db. Closes:
-    #3497
-  o Add XSalsa20 symmetric encryption method.
-  o Use pycryptopp for symmetric encryption.
-  o Add public method to access the saved password. Closes #3118.
-  o Split soledad package into common, client and server. Closes
-    #3487.
-  o Add versioneer, parse_requirements
-Server:
-  o Split soledad package into common, client and server. Closes
-    #3487.
-  o Add versioneer, parse_requirements
-Common:
-  o Split soledad package into common, client and server. Closes
-    #3487.
-  o Add versioneer, parse_requirements
-
-0.3.0 Aug 9, 2013:
-Client:
-  o Thread safe wrapper for pysqlcipher.
-  o Fix a couple of typos that prevented certain functionality to
-    work. Fixes #3306
-Server:
-  o A plaintext port is not opened by soledad server initscript call
-    to twistd web anymore. Closes #3254.
-
-0.2.3 Jul 26, 2013:
-Client:
-  o Avoid possible timing attack in document's mac comparison by
-    comparing hashes instead of plain macs. Closes #3243.
-Server:
-  o Refactor server side auth classes to make it possible for other
-    kinds of authentication to be easily implemented. Closes #2621.
-  o Fix double specified /etc/leap/soledad-server.pem in initscript by
-    pointing the PRIVKEY_PATH to /etc/leap/soledad-server.key. Fixes
-    #3174.
-
-0.2.2 Jul 12, 2013:
-Client:
-  o Add method for password change.
-Server:
-  o Use the right name as the WSGI server
-
-0.2.1 Jun 28, 2013:
-Client:
-  o Do not list the backends in the __init__'s __all__ to allow not
-    supporting couch on the client side until the code is diveded into
-    client and server.  o Fix bad dependencies in setup.py.
-  o Fix broken pip install
-  o Database request have default timeout too high, a
-    soledad.SOLEDAD_TIMEOUT variable has been added in order to have
-    more control over this. Fixes #2713
-  o Add validation and authorization of actions upon interaction with
-    server.
-  o Add MAC authentication to encrypted representation of documents.
-  o Add SQLCipher API to SQLCipher backend (allow for use of raw keys,
-    add better encrypted db assertion, add cipher, kdf_iter,
-    cipher_page_size and rekey PRAGMAS).
-  o Change symmetric encryption method to AES-256 CTR mode.
-  o Change the local storage of the storage secret:
-      * Use scrypt to derive a key for the encryption of the storage
-        secret.
-      * Store secret in a file called 'soledad.json' by default.
-      * Also store the salt and encryption details, as defined in the
-        spec.
-      * This change is not backwards compatible (i.e. all previously
-        stored secrets are incompatible with this new encryption and
-        storage scheme).
-  o Improve tests coverage.
-  o Split soledad client and server into two different packages.
-  o Use scrypt to derive the key for local encryption.
-
-Server:
-  o Add a `status` option to Soledad init script.
-  o Allow to initialize soledad with a blank server
-  o b64 encode all U1DB data in couch backend to avoid utf8 encoding
-    problems.
-    * init.d script improvements:
-    * Add LSB (Linux Standards Base) 3.1 compliant header
-    * Remove unnecessary backslashes in variable definitions
-    * Replace environment variables with more standard upper-cased names
-    * Make a TWISTD_PATH environment variable to replace hard-coded
-      /usr/local/bin/twistd
-    * Pull environment variables together into one block o Remove strict
-      dependency on leap.common.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
new file mode 100644
index 00000000..ed60b5ec
--- /dev/null
+++ b/CHANGELOG.rst
@@ -0,0 +1,55 @@
+0.8.0 - 18 Apr, 2016
+++++++++++++++++++++
+
+Client
+======
+
+Features
+~~~~~~~~
+- `#7656 <https://leap.se/code/issues/7656>`_: Emit multi-user aware events.
+- Client will now send documents at a limited size batch due to changes on SyncTarget. The default limit is 500kB. Disabled by default.
+
+Bugfixes
+~~~~~~~~
+- `#7503 <https://leap.se/code/issues/7503>`_: Do not signal sync completion if sync failed.
+- Handle missing design doc at GET (get_sync_info). Soledad server can handle this during sync.
+
+Misc
+~~~~
+- `#7195 <https://leap.se/code/issues/7195>`_: Use cryptography instead of pycryptopp.
+
+Known Issues
+~~~~~~~~~~~~
+- Upload phase of client syncs is still quite slow. Enabling size limited batching
+  can help, but you have to make sure that your server is compatible.
+
+Server
+======
+
+Features
+~~~~~~~~
+- General performance improvements.
+- `#7509 <https://leap.se/code/issues/7509>`_: Moves config directory from /etc/leap to /etc/soledad.
+- Adds a new config parameter 'create_cmd', which allows sysadmin to specify
+  which command will create a database. That command was added in
+  pkg/create-user-db and debian package automates steps needed for sudo access.
+- Read netrc path from configuration file for create-user-db command. 
+- 'create-user-db' script now can be configured from soledad-server.conf when
+  generating the user's security document.
+- Migrating a user's database to newest design documents is now possible by
+  using a parameter '--migrate-all' on 'create-user-db' script.
+- Remove tsafe monkeypatch from SSL lib, as it was needed for Twisted <12
+- Added two methods to start and finish a batch on backend. They can be used to
+  change database behaviour, allowing batch operations to be optimized.
+
+Common
+======
+
+Features
+~~~~~~~~
+- Add a sanitized command executor for database creation and re-enable user
+  database creation on CouchServerState via command line.
+
+Bugfixes
+~~~~~~~~
+- `#7626 <https://leap.se/code/issues/7626>`_: Subclass a leaky leap.common.couch exception to avoid depending on couch.
diff --git a/HISTORY b/HISTORY
new file mode 100644
index 00000000..b0530226
--- /dev/null
+++ b/HISTORY
@@ -0,0 +1,378 @@
+0.7.4 Oct 28, 2015:
+Client:
+  o Do not signal sync completion if sync failed. Closes: #7503
+  o Handle missing design doc at GET (get_sync_info). Soledad server can handle
+    this during sync.
+  o Increase http request timeout time to 90s. Related: #7382.
+  o Handle DatabaseDoesNotExist during sync.
+  o Handle MissingDesignDocError after get_sync_info.
+
+0.7.3 Sep 22, 2015:
+Client:
+  o Bugfix: refactor code loss. Closes #7412.
+  o Bugfix: Set active secret before saving local file.
+  o Split http_target into 4 modules, separating those responsibilities.
+  o Refactor details of making an HTTP request body and headers out of the
+    send/fetch logic. This also makes it easier to enable batching.
+
+Server:
+  o Fix a bug where BadRequest could be raised after everything was persisted.
+
+Common:
+  o Refactor couch.py to separate persistence from logic while saving uploaded
+    documents. Also simplify logic while checking for conflicts.
+
+
+0.7.2 Aug 26, 2015:
+Client:
+  o Remove MAC from secrets file. Closes #6980.
+  o refactor SoledadCrypto to remove circular dependency with SoledadSecrets.
+    Closes #7338.
+  o Improve how we send information on SOLEDAD_SYNC_SEND_STATUS and in
+    SOLEDAD_SYNC_RECEIVE_STATUS. Related to Feature #7353.
+  o Fix hanging sync by properly waiting db initialization on sync decrypter
+    pool. Closes #7386.
+  o Avoid double decryption of documents.
+  o Fix the order of the events emited for incoming documents.
+  o bugfix: move sync db and encpool creation to api.
+  o Bugfix: wrong sqlcipher passphrase now raises correctly.
+  o Refactor decription pool and http target to use a deferred instead of a
+    waiting loop.
+  o Improve helper scripts and dependencies listing.
+  o Refactor enc/dec pool to standardize start/stop of the pools.
+  o Add tests for enc/dec pool.
+
+Client, Server, Common:
+  o Lots of code restyling to pass CI tests.
+  o Lots of work done to get tests passing.
+  o Remove dependency on simplejson.
+
+0.7.1 Jul 10, 2015:
+Client:
+  o Fix sync encrypter pool close queue error. Closes #7088.
+  o Expose post-sync hooks via plugin system. Related: #6996.
+
+0.7.0 Jun 01 2015:
+Client:
+  o Do not depend on pysqlite2 (#2945).
+  o Reset syncer connection when getting HTTP error during sync (#5855).
+  o Wait for last post request to finish before starting a new one (#5975).
+  o Use TLS v1 in soledad client. Fixes partially #6437
+  o Retry on sqlcipher thread timeouts (#6625).
+  o Fix the order of insertion of documents when using workers for decrypting
+    incoming documents during a sync. Closes #6757.
+  o Add dependency on Twisted. Closes #6797.
+  o Fix the log message when a local secret is not found so it's less
+    confusing. Closes #6892.
+  o Always initialize the sync db to allow for both asynchronous encryption
+    and asynchronous decryption when syncing.
+  o Fallback to utf-8 if confidence on chardet guessing is too low.
+  o Refactor asynchronous encryption/decryption code to its own file.
+  o Fix logging and graceful failing when exceptions are raised during sync.
+  o Improve log messages when concurrently fetching documents from the server.
+  o Store all incoming documents in the sync db (#5895).
+  o Include the IV in the encrypted document MAC (#6400).
+  o Adapt soledad to the new events api on leap.common. Related to #6359.
+  o Add a pool of HTTP/HTTPS connections that is able to verify the server
+    certificate against a given CA certificate.
+  o Use twisted.enterprise.adbapi for access to the sync database.
+  o Use twisted.web.client for client sync.
+
+Common:
+  o Include couch design docs source files in source distribution and only
+    compile ddocs.py when building the package (#5896).
+  o Bail out if cdocs/ dir does not exist. Closes: #6671
+  o Remove unneeded parameters from CouchServerState initialization. Closes
+    #6833.
+  o  Adapt soledad to the new events api on leap.common. Related to #6359.
+
+Server:
+  o Run daemon as user soledad (#6436).
+  o Avoid use of SSLv3 (#6437).
+  o Fix server initscript location (#6557).
+  o Add dependency on Twisted. Closes #6797.
+  o Remove unneeded parameters from CouchServerState initialization. Closes
+    #6833.
+  o Fix server daemon uid and gid by passing them to twistd on the initscript.
+  o Use monthly token databases. Closes #6785.
+
+0.6.5 Apr 09 2015:
+Server:
+  o Remove unneeded parameters from CouchServerState initialization. Closes
+    #6833.
+
+0.6.4 Jan 23 2015:
+Common:
+  o Bail out if cdocs/ dir does not exist. Closes: #6671
+
+0.6.3 Dec 16 2014:
+Client:
+  o Fix deferred encryption/decryption parameters (#6500).
+
+0.6.2 Dec 11 2014:
+Client:
+  o Fix incorrect ssl context setup.
+
+0.6.1 Dec 08 2014:
+Client:
+  o Use TLS v1 in soledad client. Fixes partially #6437
+
+Server:
+  o Run daemon as user soledad (#6436).
+  o Avoid use of SSLv3 (#6437).
+
+0.6.0 Jul 18, 2014:
+Client:
+  o Close all connections after syncing. Fixes #5518.
+  o Reset synchronizer state in order to reuse the same synchronizer
+    object multiple times.
+  o Use temporal database for encryption/decryption during
+    sync. Closes #5326.
+  o Add sync status signals. Closes #5517.
+  o Allow for interrupting and recovering sync. Closes #5517.
+  o Parallelize sync requests and reuse HTTP connections.
+  o Split sync in multiple POST requests in client. Closes #5571.
+
+Common:
+  o Use a dedicated HTTP resource for couch multipart PUTs to avoid
+    bigcouch. Closes #5739.
+
+Server:
+  o Pin PyOpenSSL dependency version to <0.14 to avoid yet another
+    crypto dependency.
+  o Authenticate in time-insensitive manner. Closes #3399.
+  o Allow for interrupting and recovering sync. Closes #5517.
+  o Split sync in multiple POST requests in server. Closes #5571.
+
+0.5.2 Jun 6, 2014:
+Client:
+  o Reset synchronizer state in order to reuse the same synchronizer
+    object multiple times.
+  o Add sync status signals. Closes #5517.
+  o Allow for interrupting and recovering sync. Closes #5517.
+  o Split sync in multiple POST requests in client. Closes #5571.
+
+Common:
+  o Use a dedicated HTTP resource for couch multipart PUTs to avoid bigcouch
+    bug. Fixes #5739.
+
+Server:
+  o Allow for interrupting and recovering sync. Closes #5517.
+  o Split sync in multiple POST requests in server. Closes #5571.
+  o Authenticate in time-insensitive manner. Closes #3399.
+
+0.5.1 May 16, 2014:
+Client:
+  o Close connection with server after syncing to avoid client hanging
+    on exit. Fixes #5507.
+
+Common:
+  o Properly close connections on couch backend. Also prevent file
+    descriptor leaks on tests. Closes #5493.
+  o Prevent couch backend from always trying to create the
+    database. Fixes #5386.
+  o Prevent Couch Server State from making one uneeded GET request on
+    instantiation. Fixes #5386.
+
+0.5.0 Apr 4, 2014:
+Client:
+  o Catch lock timeout exception. Fixes #4435.
+  o Add lock for create_doc and update_indexes call, prevents
+    concurrent access to the db. Closes #5139.
+  o Back-compatibility for socket.create_connection interface in
+    2.6. Closes #5208.
+  o Always return unicode in helper method, even on
+    UnicodeError. Related to #4998.
+  o Fix a bug in soledad.client.sqlcipher by which we were creating a
+    new connection for each sync.
+  o Unlock shared_db if anything fails in the bootstrap
+    sequence. Fixes #4702.
+  o Avoid concurrent syncs for the same account, but allow for
+    distinct accounts. Fixes #4451.
+  o Adds a get_count_by_index to sqlcipher u1db backend. Related to:
+    #4616.
+  o Do not autocreate remote user database when syncing. Tapicero
+    should make sure that that db is created when the user is
+    created. Closes #5302.
+  o Add a read-write lock for all client operations. Addresses: #4972
+  o Add sync=off and tem_store=mem to soledad client, for
+    optimization.
+
+Common:
+  o Add lock timeout HTTP error. Fixes #4435.
+  o Remodel couch backend to fix concurrency and scalability. Closes
+    #4475, #4682, #4683 and #4680.
+  o Remove check for design docs on couch server state initialization
+    Closes #5387.
+  o Renew HTTP session after multipart PUTs to avoid request hanging.
+    Fixes #5449.
+  o Preload time.strptime() to avoid multi-threaded problem on couch
+    backend get_docs() method. Fixes #5449.
+  o Improve error messages. Closes #5035.
+  o Add MissingTokenError and InvalidTokenError as sub exceptions
+    from Unauthorized.
+  o Allow sync of large files (~100MB). Closes #4836.
+  o Add exceptions to deal with missing design documents. Fixes #4994.
+  o Parallelize get_docs() on couch backend to accelerate sync.
+    Closes #5008.
+  o Use less memory when putting docs on couch. Fixes #5011.
+  o Prevent CouchServerState from creating or deleting databases. This
+    way, Soledad remote clients won't ever be able to do these
+    operations when syncing. Part of #5302.
+  o Avoid concurrent syncs problem by adding a lock for PUTting to the
+    sync log update handler. Fixes #5388.
+  o Remove check for couch permissions when CouchServerState is
+    instantiated. This is not necessary anymore because platform
+    takes care of giving the soledad user enough permissions and
+    tapicero takes care of uploading the needed design documents.
+
+Server:
+  o Send propper lock timeout response. Fixes #4435.
+  o Fix raising of auth token errors. Fixes #5191.
+  o Allow sync of large files (~100MB). Closes #4836.
+  o Use a temporary directory for server side locks. Fixes #4918.
+  o Catch couchdb.http.ResourceNotFound exceptions when accessing
+    design documents on couch backend, and raise appropriate missing
+    design documents exceptions. Fixes #4994.
+  o Do not try to create the shared database when running the Soledad
+    Server application. Fixes #5302.
+  o Enable Gzip compression on the soledad wsgi app.
+
+0.4.4 Dec 6, 2013:
+Client:
+  o Add MAC verirication to the recovery document and
+    soledad.json. Closes #4348.
+Common:
+  o Add unicode conversion to put_doc(). Closes #4095.
+  o Remove tests dependency on nose2. Closes #4258.
+
+0.4.3 Nov 15, 2013:
+Client:
+  o Defaults detected encoding to utf-8 to avoid bug if detected
+    encoding is None. Closes: #4417
+  o Open db in autocommit mode, to avoid nested transactions problems.
+    Closes: #4400
+
+0.4.2 Nov 1, 2013:
+Client:
+  o Support non-ascii passwords. Closes #4001.
+  o Change error severity for missing secrets path.
+  o Use chardet as fallback if cchardet not found.
+  o Improve bootstrap sequence and allow for locking the shared
+    database while creating/uploading the encryption secret. Closes
+    #4097.
+Common:
+  o Move some common functions and global variables to
+    leap.soledad.common.
+Server:
+  o Allow for locking the shared database. Closes #4097.
+
+0.4.1 Oct 4, 2013:
+Client:
+  o Save only UTF8 strings. Related to #3660.
+
+0.4.0 Sep 20, 2013:
+Client:
+  o Remove redundant logging when creating data dirs.
+Server:
+  o Verify for couch permissions when starting server. Closes #3501.
+Common:
+  o Improve u1db data storage in couch. Closes #3647.
+  o Turn couchdb dependency for common into optional. Closes #2167.
+  o Add verification for couch permissions. Closes #3501.
+
+0.3.2 Sep 6, 2013:
+Client:
+  o Use dirspec instead of plain xdg. Closes #3574.
+Server:
+  o Fix the init script for soledad server so that it uses the new
+    package namespace.
+
+0.3.1 Aug 23, 2013:
+Client:
+  o Add libsqlite3-dev requirement for soledad.
+  o Check for None in private methods that depend on _db. Closes:
+    #3497
+  o Add XSalsa20 symmetric encryption method.
+  o Use pycryptopp for symmetric encryption.
+  o Add public method to access the saved password. Closes #3118.
+  o Split soledad package into common, client and server. Closes
+    #3487.
+  o Add versioneer, parse_requirements
+Server:
+  o Split soledad package into common, client and server. Closes
+    #3487.
+  o Add versioneer, parse_requirements
+Common:
+  o Split soledad package into common, client and server. Closes
+    #3487.
+  o Add versioneer, parse_requirements
+
+0.3.0 Aug 9, 2013:
+Client:
+  o Thread safe wrapper for pysqlcipher.
+  o Fix a couple of typos that prevented certain functionality to
+    work. Fixes #3306
+Server:
+  o A plaintext port is not opened by soledad server initscript call
+    to twistd web anymore. Closes #3254.
+
+0.2.3 Jul 26, 2013:
+Client:
+  o Avoid possible timing attack in document's mac comparison by
+    comparing hashes instead of plain macs. Closes #3243.
+Server:
+  o Refactor server side auth classes to make it possible for other
+    kinds of authentication to be easily implemented. Closes #2621.
+  o Fix double specified /etc/leap/soledad-server.pem in initscript by
+    pointing the PRIVKEY_PATH to /etc/leap/soledad-server.key. Fixes
+    #3174.
+
+0.2.2 Jul 12, 2013:
+Client:
+  o Add method for password change.
+Server:
+  o Use the right name as the WSGI server
+
+0.2.1 Jun 28, 2013:
+Client:
+  o Do not list the backends in the __init__'s __all__ to allow not
+    supporting couch on the client side until the code is diveded into
+    client and server.  o Fix bad dependencies in setup.py.
+  o Fix broken pip install
+  o Database request have default timeout too high, a
+    soledad.SOLEDAD_TIMEOUT variable has been added in order to have
+    more control over this. Fixes #2713
+  o Add validation and authorization of actions upon interaction with
+    server.
+  o Add MAC authentication to encrypted representation of documents.
+  o Add SQLCipher API to SQLCipher backend (allow for use of raw keys,
+    add better encrypted db assertion, add cipher, kdf_iter,
+    cipher_page_size and rekey PRAGMAS).
+  o Change symmetric encryption method to AES-256 CTR mode.
+  o Change the local storage of the storage secret:
+      * Use scrypt to derive a key for the encryption of the storage
+        secret.
+      * Store secret in a file called 'soledad.json' by default.
+      * Also store the salt and encryption details, as defined in the
+        spec.
+      * This change is not backwards compatible (i.e. all previously
+        stored secrets are incompatible with this new encryption and
+        storage scheme).
+  o Improve tests coverage.
+  o Split soledad client and server into two different packages.
+  o Use scrypt to derive the key for local encryption.
+
+Server:
+  o Add a `status` option to Soledad init script.
+  o Allow to initialize soledad with a blank server
+  o b64 encode all U1DB data in couch backend to avoid utf8 encoding
+    problems.
+    * init.d script improvements:
+    * Add LSB (Linux Standards Base) 3.1 compliant header
+    * Remove unnecessary backslashes in variable definitions
+    * Replace environment variables with more standard upper-cased names
+    * Make a TWISTD_PATH environment variable to replace hard-coded
+      /usr/local/bin/twistd
+    * Pull environment variables together into one block o Remove strict
+      dependency on leap.common.
diff --git a/client/changes/bug_7503-do-not-signal-sync-complete b/client/changes/bug_7503-do-not-signal-sync-complete
deleted file mode 100644
index 4cc361e0..00000000
--- a/client/changes/bug_7503-do-not-signal-sync-complete
+++ /dev/null
@@ -1 +0,0 @@
-o Do not signal sync completion if sync failed. Closes: #7503
diff --git a/client/changes/bug_missing_design_doc_handler b/client/changes/bug_missing_design_doc_handler
deleted file mode 100644
index 72e42b85..00000000
--- a/client/changes/bug_missing_design_doc_handler
+++ /dev/null
@@ -1 +0,0 @@
-o Handle missing design doc at GET (get_sync_info). Soledad server can handle this during sync.
diff --git a/client/changes/feat_send_batch b/client/changes/feat_send_batch
deleted file mode 100644
index fbfce519..00000000
--- a/client/changes/feat_send_batch
+++ /dev/null
@@ -1 +0,0 @@
-o Client will now send documents at a limited size batch due to changes on SyncTarget. The default limit is 500kB.
diff --git a/client/changes/feat_use_cryptography b/client/changes/feat_use_cryptography
deleted file mode 100644
index 6e8fe3bf..00000000
--- a/client/changes/feat_use_cryptography
+++ /dev/null
@@ -1 +0,0 @@
-o Use cryptography instead of pycryptopp. Stick with AES-CTR.
diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst
index 89234cb2..bdc9f893 100644
--- a/client/changes/next-changelog.rst
+++ b/client/changes/next-changelog.rst
@@ -1,5 +1,5 @@
-0.8.0 - xxx
-+++++++++++++++++++++++++++++++
+0.8.1 - ...
+++++++++++++++++++++
 
 Please add lines to this file, they will be moved to the CHANGELOG.rst during
 the next release.
@@ -10,7 +10,6 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff.
 
 Features
 ~~~~~~~~
-- `#7656 <https://leap.se/code/issues/7656>`_: Emit multi-user aware events.
 - `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
 - New feature without related issue number.
 
@@ -21,7 +20,6 @@ Bugfixes
 
 Misc
 ~~~~
-- `#7195 <https://leap.se/code/issues/7195>`_: Use cryptography instead of pycryptopp.
 - `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
 - Some change without issue number.
 
diff --git a/common/changes/create_db_cmd b/common/changes/create_db_cmd
deleted file mode 100644
index 00bbdf71..00000000
--- a/common/changes/create_db_cmd
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Add a sanitized command executor for database creation and re-enable
-  user database creation on CouchServerState via command line.
diff --git a/common/changes/next-changelog.rst b/common/changes/next-changelog.rst
index e05f80ce..c0974384 100644
--- a/common/changes/next-changelog.rst
+++ b/common/changes/next-changelog.rst
@@ -1,4 +1,4 @@
-0.8.0 - xxx
+0.8.0 - ...
 +++++++++++++++++++++++++++++++
 
 Please add lines to this file, they will be moved to the CHANGELOG.rst during
@@ -15,7 +15,6 @@ Features
 
 Bugfixes
 ~~~~~~~~
-- `#7626 <https://leap.se/code/issues/7626>`_: Subclass a leaky leap.common.couch exception to avoid depending on couch.
 - `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235.
 - Bugfix without related issue number.
 
diff --git a/server/changes/change_soledad_configdir b/server/changes/change_soledad_configdir
deleted file mode 100644
index 710b9ac8..00000000
--- a/server/changes/change_soledad_configdir
+++ /dev/null
@@ -1,2 +0,0 @@
-o Moves config directory from /etc/leap to /etc/soledad
-  resolves #7509
diff --git a/server/changes/create_db_cmd b/server/changes/create_db_cmd
deleted file mode 100644
index 964a7906..00000000
--- a/server/changes/create_db_cmd
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Adds a new config parameter 'create_cmd', which allows sysadmin to specify
-  which command will create a database. That command was added in
-  pkg/create-user-db and debian package automates steps needed for sudo access.
-  o Read netrc path from configuration file for create-user-db command. 
diff --git a/server/changes/feat_configurable_ensure b/server/changes/feat_configurable_ensure
deleted file mode 100644
index 8abd2ac9..00000000
--- a/server/changes/feat_configurable_ensure
+++ /dev/null
@@ -1,5 +0,0 @@
-o 'create-user-db' script now can be configured from soledad-server.conf
-  when generating the user's security document.
-o Migrating a user's database to newest design documents is now possible by
-  using a parameter '--migrate-all' on 'create-user-db' script.
-o Remove tsafe monkeypatch from SSL lib, as it was needed for Twisted <12
diff --git a/server/changes/feat_handle_send_batch_better b/server/changes/feat_handle_send_batch_better
deleted file mode 100644
index 6ee8688a..00000000
--- a/server/changes/feat_handle_send_batch_better
+++ /dev/null
@@ -1 +0,0 @@
-o Added two methods to start and finish a batch on backend. They can be used to change database behaviour, allowing batch operations to be optimized.
diff --git a/server/changes/next-changelog.rst b/server/changes/next-changelog.rst
new file mode 100644
index 00000000..bdc9f893
--- /dev/null
+++ b/server/changes/next-changelog.rst
@@ -0,0 +1,28 @@
+0.8.1 - ...
+++++++++++++++++++++
+
+Please add lines to this file, they will be moved to the CHANGELOG.rst during
+the next release.
+
+There are two template lines for each category, use them as reference.
+
+I've added a new category `Misc` so we can track doc/style/packaging stuff.
+
+Features
+~~~~~~~~
+- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
+- New feature without related issue number.
+
+Bugfixes
+~~~~~~~~
+- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235.
+- Bugfix without related issue number.
+
+Misc
+~~~~
+- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
+- Some change without issue number.
+
+Known Issues
+~~~~~~~~~~~~
+- `#1236 <https://leap.se/code/issues/1236>`_: Description of the known issue corresponding with issue #1236.
-- 
cgit v1.2.3