From a4872bd64884cccd85e63cc2cae631f26dbc96c4 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 27 Jan 2016 20:45:00 -0300
Subject: [feat] defer blocking requests calls to thread

That's a temporary fix for #6506
This commit adapts code to deal with deferreds coming from calling
requests from Twisted. Next step is just to change requests for twisted
http client present in leap.common.
Unfortunately, this last step will be a bit longer and would be better
to have integrations tests to ensure current HTTP behaviour.
---
 src/leap/keymanager/__init__.py | 42 +++++++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index c7886e0..06128c0 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -58,6 +58,7 @@ import logging
 import requests
 
 from twisted.internet import defer
+from twisted.internet import threads
 from urlparse import urlparse
 
 from leap.common.check import leap_assert
@@ -185,6 +186,7 @@ class KeyManager(object):
             lambda klass: klass.__name__ == ktype,
             self._wrapper_map).pop()
 
+    @defer.inlineCallbacks
     def _get(self, uri, data=None):
         """
         Send a GET request to C{uri} containing C{data}.
@@ -200,7 +202,8 @@ class KeyManager(object):
         leap_assert(
             self._ca_cert_path is not None,
             'We need the CA certificate path!')
-        res = self._fetcher.get(uri, data=data, verify=self._ca_cert_path)
+        res = yield threads.deferToThread(self._fetcher.get, uri, data=data,
+                                          verify=self._ca_cert_path)
         # Nickserver now returns 404 for key not found and 500 for
         # other cases (like key too small), so we are skipping this
         # check for the time being
@@ -211,7 +214,7 @@ class KeyManager(object):
         # leap_assert(
         #     res.headers['content-type'].startswith('application/json'),
         #     'Content-type is not JSON.')
-        return res
+        defer.returnValue(res)
 
     def _get_with_combined_ca_bundle(self, uri, data=None):
         """
@@ -228,9 +231,11 @@ class KeyManager(object):
         :return: The response to the request.
         :rtype: requests.Response
         """
-        return self._fetcher.get(
-            uri, data=data, verify=self._combined_ca_bundle)
+        return threads.deferToThread(self._fetcher.get,
+                                     uri, data=data,
+                                     verify=self._combined_ca_bundle)
 
+    @defer.inlineCallbacks
     def _put(self, uri, data=None):
         """
         Send a PUT request to C{uri} containing C{data}.
@@ -253,14 +258,17 @@ class KeyManager(object):
         leap_assert(
             self._token is not None,
             'We need a token to interact with webapp!')
-        res = self._fetcher.put(
-            uri, data=data, verify=self._ca_cert_path,
-            headers={'Authorization': 'Token token=%s' % self._token})
+        headers = {'Authorization': 'Token token=%s' % self._token}
+        res = yield threads.deferToThread(self._fetcher.put,
+                                          uri, data=data,
+                                          verify=self._ca_cert_path,
+                                          headers=headers)
         # assert that the response is valid
         res.raise_for_status()
-        return res
+        defer.returnValue(res)
 
     @memoized_method(invalidation=300)
+    @defer.inlineCallbacks
     def _fetch_keys_from_server(self, address):
         """
         Fetch keys bound to address from nickserver and insert them in
@@ -279,7 +287,7 @@ class KeyManager(object):
         d = defer.succeed(None)
         res = None
         try:
-            res = self._get(self._nickserver_uri, {'address': address})
+            res = yield self._get(self._nickserver_uri, {'address': address})
             res.raise_for_status()
             server_keys = res.json()
 
@@ -307,7 +315,7 @@ class KeyManager(object):
         except Exception as e:
             d = defer.fail(KeyNotFound(e.message))
             logger.warning("Error retrieving key: %r" % (e,))
-        return d
+        yield d
 
     #
     # key management
@@ -339,8 +347,9 @@ class KeyManager(object):
                 self._api_uri,
                 self._api_version,
                 self._uid)
-            self._put(uri, data)
+            d = self._put(uri, data)
             emit_async(catalog.KEYMANAGER_DONE_UPLOADING_KEYS, self._address)
+            return d
 
         d = self.get_key(
             self._address, ktype, private=False, fetch_remote=False)
@@ -822,6 +831,7 @@ class KeyManager(object):
             d.addCallback(lambda _: self.put_key(privkey, address))
         return d
 
+    @defer.inlineCallbacks
     def fetch_key(self, address, uri, ktype,
                   validation=ValidationLevels.Weak_Chain):
         """
@@ -852,20 +862,20 @@ class KeyManager(object):
 
         logger.info("Fetch key for %s from %s" % (address, uri))
         try:
-            res = self._get_with_combined_ca_bundle(uri)
+            res = yield self._get_with_combined_ca_bundle(uri)
         except Exception as e:
             logger.warning("There was a problem fetching key: %s" % (e,))
-            return defer.fail(KeyNotFound(uri))
+            raise KeyNotFound(uri)
         if not res.ok:
-            return defer.fail(KeyNotFound(uri))
+            raise KeyNotFound(uri)
 
         # XXX parse binary keys
         pubkey, _ = _keys.parse_ascii_key(res.content)
         if pubkey is None:
-            return defer.fail(KeyNotFound(uri))
+            raise KeyNotFound(uri)
 
         pubkey.validation = validation
-        return self.put_key(pubkey, address)
+        yield self.put_key(pubkey, address)
 
     def _assert_supported_key_type(self, ktype):
         """
-- 
cgit v1.2.3


From e759ab61e8a22235c45192750b536612a6c2cb8a Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 27 Jan 2016 21:35:43 -0300
Subject: [refactor] isolate requests

Isolate requests lib related code and update docstrings.
---
 src/leap/keymanager/__init__.py              | 112 +++++++++++++--------------
 src/leap/keymanager/tests/test_keymanager.py |   8 +-
 2 files changed, 56 insertions(+), 64 deletions(-)

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 06128c0..6413e9e 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -187,35 +187,43 @@ class KeyManager(object):
             self._wrapper_map).pop()
 
     @defer.inlineCallbacks
-    def _get(self, uri, data=None):
+    def _get_json(self, address):
         """
         Send a GET request to C{uri} containing C{data}.
 
         :param uri: The URI of the request.
         :type uri: str
-        :param data: The body of the request.
-        :type data: dict, str or file
 
-        :return: The response to the request.
-        :rtype: requests.Response
+        :return: A deferred that will be fired with the GET response
+        :rtype: Deferred
         """
         leap_assert(
             self._ca_cert_path is not None,
             'We need the CA certificate path!')
-        res = yield threads.deferToThread(self._fetcher.get, uri, data=data,
-                                          verify=self._ca_cert_path)
-        # Nickserver now returns 404 for key not found and 500 for
-        # other cases (like key too small), so we are skipping this
-        # check for the time being
-        # res.raise_for_status()
-
+        try:
+            uri, data = self._nickserver_uri, {'address': address}
+            res = yield threads.deferToThread(self._fetcher.get, uri,
+                                              data=data,
+                                              verify=self._ca_cert_path)
+            res.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                raise KeyNotFound(address)
+            else:
+                raise KeyNotFound(e.message)
+            logger.warning("HTTP error retrieving key: %r" % (e,))
+            logger.warning("%s" % (res.content,))
+        except Exception as e:
+            raise KeyNotFound(e.message)
+            logger.warning("Error retrieving key: %r" % (e,))
         # Responses are now text/plain, although it's json anyway, but
         # this will fail when it shouldn't
         # leap_assert(
         #     res.headers['content-type'].startswith('application/json'),
         #     'Content-type is not JSON.')
-        defer.returnValue(res)
+        defer.returnValue(res.json())
 
+    @defer.inlineCallbacks
     def _get_with_combined_ca_bundle(self, uri, data=None):
         """
         Send a GET request to C{uri} containing C{data}.
@@ -228,12 +236,18 @@ class KeyManager(object):
         :param data: The body of the request.
         :type data: dict, str or file
 
-        :return: The response to the request.
-        :rtype: requests.Response
+        :return: A deferred that will be fired with the GET response
+        :rtype: Deferred
         """
-        return threads.deferToThread(self._fetcher.get,
-                                     uri, data=data,
-                                     verify=self._combined_ca_bundle)
+        try:
+            res = yield threads.deferToThread(self._fetcher.get, uri,
+                                              verify=self._combined_ca_bundle)
+        except Exception as e:
+            logger.warning("There was a problem fetching key: %s" % (e,))
+            raise KeyNotFound(uri)
+        if not res.ok:
+            raise KeyNotFound(uri)
+        defer.returnValue(res.content)
 
     @defer.inlineCallbacks
     def _put(self, uri, data=None):
@@ -249,8 +263,8 @@ class KeyManager(object):
         :param data: The body of the request.
         :type data: dict, str or file
 
-        :return: The response to the request.
-        :rtype: requests.Response
+        :return: A deferred that will be fired when PUT request finishes
+        :rtype: Deferred
         """
         leap_assert(
             self._ca_cert_path is not None,
@@ -284,38 +298,22 @@ class KeyManager(object):
 
         """
         # request keys from the nickserver
-        d = defer.succeed(None)
-        res = None
-        try:
-            res = yield self._get(self._nickserver_uri, {'address': address})
-            res.raise_for_status()
-            server_keys = res.json()
-
-            # insert keys in local database
-            if self.OPENPGP_KEY in server_keys:
-                # nicknym server is authoritative for its own domain,
-                # for other domains the key might come from key servers.
-                validation_level = ValidationLevels.Weak_Chain
-                _, domain = _split_email(address)
-                if (domain == _get_domain(self._nickserver_uri)):
-                    validation_level = ValidationLevels.Provider_Trust
-
-                d = self.put_raw_key(
-                    server_keys['openpgp'],
-                    OpenPGPKey,
-                    address=address,
-                    validation=validation_level)
-        except requests.exceptions.HTTPError as e:
-            if e.response.status_code == 404:
-                d = defer.fail(KeyNotFound(address))
-            else:
-                d = defer.fail(KeyNotFound(e.message))
-            logger.warning("HTTP error retrieving key: %r" % (e,))
-            logger.warning("%s" % (res.content,))
-        except Exception as e:
-            d = defer.fail(KeyNotFound(e.message))
-            logger.warning("Error retrieving key: %r" % (e,))
-        yield d
+        server_keys = yield self._get_json(address)
+
+        # insert keys in local database
+        if self.OPENPGP_KEY in server_keys:
+            # nicknym server is authoritative for its own domain,
+            # for other domains the key might come from key servers.
+            validation_level = ValidationLevels.Weak_Chain
+            _, domain = _split_email(address)
+            if (domain == _get_domain(self._nickserver_uri)):
+                validation_level = ValidationLevels.Provider_Trust
+
+            yield self.put_raw_key(
+                server_keys['openpgp'],
+                OpenPGPKey,
+                address=address,
+                validation=validation_level)
 
     #
     # key management
@@ -861,16 +859,10 @@ class KeyManager(object):
         _keys = self._wrapper_map[ktype]
 
         logger.info("Fetch key for %s from %s" % (address, uri))
-        try:
-            res = yield self._get_with_combined_ca_bundle(uri)
-        except Exception as e:
-            logger.warning("There was a problem fetching key: %s" % (e,))
-            raise KeyNotFound(uri)
-        if not res.ok:
-            raise KeyNotFound(uri)
+        ascii_content = yield self._get_with_combined_ca_bundle(uri)
 
         # XXX parse binary keys
-        pubkey, _ = _keys.parse_ascii_key(res.content)
+        pubkey, _ = _keys.parse_ascii_key(ascii_content)
         if pubkey is None:
             raise KeyNotFound(uri)
 
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index 856d6da..afcbd5f 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -342,7 +342,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL, data=None,
+        get_mock.assert_called_once_with(REMOTE_KEY_URL,
                                          verify=ca_bundle.where())
 
     @inlineCallbacks
@@ -353,7 +353,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL, data=None,
+        get_mock.assert_called_once_with(REMOTE_KEY_URL,
                                          verify=ca_bundle.where())
 
     @inlineCallbacks
@@ -364,7 +364,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL, data=None,
+        get_mock.assert_called_once_with(REMOTE_KEY_URL,
                                          verify=ca_bundle.where())
 
     @inlineCallbacks
@@ -383,7 +383,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
                 yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
                 # assert that combined bundle file is passed to get call
-                get_mock.assert_called_once_with(REMOTE_KEY_URL, data=None,
+                get_mock.assert_called_once_with(REMOTE_KEY_URL,
                                                  verify=tmp_output.name)
 
                 # assert that files got appended
-- 
cgit v1.2.3


From cb32761d3dc97956085cd34a8f1e2e06432e8141 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 27 Jan 2016 23:18:04 -0300
Subject: [feat] use HTTPClient instead of requests

This commit adapts code to use HTTPClient instead of requests.
requests library receives a certificate as parameter during requests
while HTTPClient recelives a cert only on constructor. In order to have
both types (leap cert and commercial certs) working together we
introduced two clients on constructor.
---
 src/leap/keymanager/__init__.py              |  63 +++++++++-------
 src/leap/keymanager/tests/test_keymanager.py | 109 +++++++++++----------------
 2 files changed, 82 insertions(+), 90 deletions(-)

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 6413e9e..1dcf642 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -22,6 +22,8 @@ import fileinput
 import os
 import sys
 import tempfile
+import json
+import urllib
 
 from leap.common import ca_bundle
 
@@ -58,10 +60,10 @@ import logging
 import requests
 
 from twisted.internet import defer
-from twisted.internet import threads
 from urlparse import urlparse
 
 from leap.common.check import leap_assert
+from leap.common.http import HTTPClient
 from leap.common.events import emit_async, catalog
 from leap.common.decorators import memoized_method
 
@@ -142,6 +144,8 @@ class KeyManager(object):
         # the following are used to perform https requests
         self._fetcher = requests
         self._combined_ca_bundle = self._create_combined_bundle_file()
+        self._async_client = HTTPClient(self._combined_ca_bundle)
+        self._async_client_pinned = HTTPClient(self._ca_cert_path)
 
     #
     # destructor
@@ -201,27 +205,29 @@ class KeyManager(object):
             self._ca_cert_path is not None,
             'We need the CA certificate path!')
         try:
-            uri, data = self._nickserver_uri, {'address': address}
-            res = yield threads.deferToThread(self._fetcher.get, uri,
-                                              data=data,
-                                              verify=self._ca_cert_path)
-            res.raise_for_status()
-        except requests.exceptions.HTTPError as e:
-            if e.response.status_code == 404:
-                raise KeyNotFound(address)
-            else:
-                raise KeyNotFound(e.message)
+            uri = self._nickserver_uri + '?address=' + address
+            content = yield self._async_client_pinned.request(str(uri), 'GET')
+            json_content = json.loads(content)
+        except IOError as e:
+            # FIXME: 404 doesnt raise today, but it wont produce json anyway
+            # if e.response.status_code == 404:
+                # raise KeyNotFound(address)
             logger.warning("HTTP error retrieving key: %r" % (e,))
-            logger.warning("%s" % (res.content,))
+            logger.warning("%s" % (content,))
+            raise KeyNotFound(e), None, sys.exc_info()[2]
+        except ValueError as v:
+            logger.warning("Invalid JSON data from key: %s" % (uri,))
+            raise KeyNotFound(v.message + ' - ' + uri), None, sys.exc_info()[2]
+
         except Exception as e:
-            raise KeyNotFound(e.message)
             logger.warning("Error retrieving key: %r" % (e,))
+            raise KeyNotFound(e.message), None, sys.exc_info()[2]
         # Responses are now text/plain, although it's json anyway, but
         # this will fail when it shouldn't
         # leap_assert(
         #     res.headers['content-type'].startswith('application/json'),
         #     'Content-type is not JSON.')
-        defer.returnValue(res.json())
+        defer.returnValue(json_content)
 
     @defer.inlineCallbacks
     def _get_with_combined_ca_bundle(self, uri, data=None):
@@ -240,14 +246,13 @@ class KeyManager(object):
         :rtype: Deferred
         """
         try:
-            res = yield threads.deferToThread(self._fetcher.get, uri,
-                                              verify=self._combined_ca_bundle)
+            content = yield self._async_client.request(str(uri), 'GET')
         except Exception as e:
             logger.warning("There was a problem fetching key: %s" % (e,))
             raise KeyNotFound(uri)
-        if not res.ok:
+        if not content:
             raise KeyNotFound(uri)
-        defer.returnValue(res.content)
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     def _put(self, uri, data=None):
@@ -272,14 +277,20 @@ class KeyManager(object):
         leap_assert(
             self._token is not None,
             'We need a token to interact with webapp!')
-        headers = {'Authorization': 'Token token=%s' % self._token}
-        res = yield threads.deferToThread(self._fetcher.put,
-                                          uri, data=data,
-                                          verify=self._ca_cert_path,
-                                          headers=headers)
-        # assert that the response is valid
-        res.raise_for_status()
-        defer.returnValue(res)
+        if type(data) == dict:
+            data = urllib.urlencode(data)
+        headers = {'Authorization': [str('Token token=%s' % self._token)]}
+        headers['Content-Type'] = ['application/x-www-form-urlencoded']
+        try:
+            res = yield self._async_client_pinned.request(str(uri), 'PUT', body=str(data), headers=headers)
+        except Exception as e:
+            logger.warning("Error uploading key: %r" % (e,))
+            raise e
+        if 'error' in res:
+            # FIXME: That's a workaround for 500,
+            # we need to implement a readBody to assert response code
+            logger.warning("Error uploading key: %r" % (res,))
+            raise Exception(res)
 
     @memoized_method(invalidation=300)
     @defer.inlineCallbacks
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index afcbd5f..c3426b6 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -21,11 +21,14 @@ Tests for the Key Manager.
 """
 
 from os import path
+import json
+import urllib
 from datetime import datetime
 import tempfile
+import pkg_resources
 from leap.common import ca_bundle
 from mock import Mock, MagicMock, patch
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet import defer
 from twisted.trial import unittest
 
 from leap.keymanager import (
@@ -127,7 +130,7 @@ class KeyManagerUtilTestCase(unittest.TestCase):
 
 class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_all_keys_in_db(self):
         km = self._key_manager()
         yield km._wrapper_map[OpenPGPKey].put_ascii_key(PRIVATE_KEY, ADDRESS)
@@ -142,7 +145,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         self.assertTrue(ADDRESS in keys[0].address)
         self.assertTrue(keys[0].private)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_public_key(self):
         km = self._key_manager()
         yield km._wrapper_map[OpenPGPKey].put_ascii_key(PRIVATE_KEY, ADDRESS)
@@ -155,7 +158,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
             key.fingerprint.lower(), KEY_FINGERPRINT.lower())
         self.assertFalse(key.private)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_private_key(self):
         km = self._key_manager()
         yield km._wrapper_map[OpenPGPKey].put_ascii_key(PRIVATE_KEY, ADDRESS)
@@ -173,7 +176,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         d = km.send_key(OpenPGPKey)
         return self.assertFailure(d, KeyNotFound)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_send_key(self):
         """
         Test that request is well formed when sending keys to server.
@@ -181,7 +184,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         token = "mytoken"
         km = self._key_manager(token=token)
         yield km._wrapper_map[OpenPGPKey].put_ascii_key(PUBLIC_KEY, ADDRESS)
-        km._fetcher.put = Mock()
+        km._async_client_pinned.request = Mock(return_value=defer.succeed(''))
         # the following data will be used on the send
         km.ca_cert_path = 'capath'
         km.session_id = 'sessionid'
@@ -191,13 +194,15 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         yield km.send_key(OpenPGPKey)
         # setup expected args
         pubkey = yield km.get_key(km._address, OpenPGPKey)
-        data = {
+        data = urllib.urlencode({
             km.PUBKEY_KEY: pubkey.key_data,
-        }
+        })
+        headers = {'Authorization': [str('Token token=%s' % token)]}
+        headers['Content-Type'] = ['application/x-www-form-urlencoded']
         url = '%s/%s/users/%s.json' % ('apiuri', 'apiver', 'myuid')
-        km._fetcher.put.assert_called_once_with(
-            url, data=data, verify='capath',
-            headers={'Authorization': 'Token token=%s' % token},
+        km._async_client_pinned.request.assert_called_once_with(
+            str(url), 'PUT', body=str(data),
+            headers=headers
         )
 
     def test_fetch_keys_from_server(self):
@@ -205,19 +210,19 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         Test that the request is well formed when fetching keys from server.
         """
         km = self._key_manager(url=NICKSERVER_URI)
+        expected_url = NICKSERVER_URI + '?address=' + ADDRESS_2
 
         def verify_the_call(_):
-            km._fetcher.get.assert_called_once_with(
-                NICKSERVER_URI,
-                data={'address': ADDRESS_2},
-                verify='cacertpath',
+            km._async_client_pinned.request.assert_called_once_with(
+                expected_url,
+                'GET',
             )
 
         d = self._fetch_key(km, ADDRESS_2, PUBLIC_KEY_2)
         d.addCallback(verify_the_call)
         return d
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_key_fetches_from_server(self):
         """
         Test that getting a key successfuly fetches from server.
@@ -229,7 +234,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         self.assertTrue(ADDRESS in key.address)
         self.assertEqual(key.validation, ValidationLevels.Provider_Trust)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_key_fetches_other_domain(self):
         """
         Test that getting a key successfuly fetches from server.
@@ -245,18 +250,10 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         """
         :returns: a Deferred that will fire with the OpenPGPKey
         """
-        class Response(object):
-            status_code = 200
-            headers = {'content-type': 'application/json'}
-
-            def json(self):
-                return {'address': address, 'openpgp': key}
-
-            def raise_for_status(self):
-                pass
+        data = json.dumps({'address': address, 'openpgp': key})
 
         # mock the fetcher so it returns the key for ADDRESS_2
-        km._fetcher.get = Mock(return_value=Response())
+        km._async_client_pinned.request = Mock(return_value=defer.succeed(data))
         km.ca_cert_path = 'cacertpath'
         # try to key get without fetching from server
         d_fail = km.get_key(address, OpenPGPKey, fetch_remote=False)
@@ -265,7 +262,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         d.addCallback(lambda _: km.get_key(address, OpenPGPKey))
         return d
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_put_key_ascii(self):
         """
         Test that putting ascii key works
@@ -277,7 +274,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         self.assertIsInstance(key, OpenPGPKey)
         self.assertTrue(ADDRESS in key.address)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_uri_ascii_key(self):
         """
         Test that fetch key downloads the ascii key and gets included in
@@ -285,11 +282,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         """
         km = self._key_manager()
 
-        class Response(object):
-            ok = True
-            content = PUBLIC_KEY
-
-        km._fetcher.get = Mock(return_value=Response())
+        km._async_client.request = Mock(return_value=defer.succeed(PUBLIC_KEY))
 
         yield km.fetch_key(ADDRESS, "http://site.domain/key", OpenPGPKey)
         key = yield km.get_key(ADDRESS, OpenPGPKey)
@@ -316,25 +309,16 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         """
         km = self._key_manager()
 
-        class Response(object):
-            ok = True
-            content = PUBLIC_KEY
-
-        km._fetcher.get = Mock(return_value=Response())
+        km._async_client.request = Mock(return_value=defer.succeed(PUBLIC_KEY))
         d = km.fetch_key(ADDRESS_2, "http://site.domain/key", OpenPGPKey)
         return self.assertFailure(d, KeyAddressMismatch)
 
     def _mock_get_response(self, km, body):
-        class Response(object):
-            ok = True
-            content = body
-
-        mock = MagicMock(return_value=Response())
-        km._fetcher.get = mock
+        km._async_client.request = MagicMock(return_value=defer.succeed(body))
 
-        return mock
+        return km._async_client.request
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_key_uses_ca_bundle_if_none_specified(self):
         ca_cert_path = None
         km = self._key_manager(ca_cert_path=ca_cert_path)
@@ -342,10 +326,9 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL,
-                                         verify=ca_bundle.where())
+        get_mock.assert_called_once_with(REMOTE_KEY_URL, 'GET')
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_key_uses_ca_bundle_if_empty_string_specified(self):
         ca_cert_path = ''
         km = self._key_manager(ca_cert_path=ca_cert_path)
@@ -353,10 +336,9 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL,
-                                         verify=ca_bundle.where())
+        get_mock.assert_called_once_with(REMOTE_KEY_URL, 'GET')
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_key_use_default_ca_bundle_if_set_as_ca_cert_path(self):
         ca_cert_path = ca_bundle.where()
         km = self._key_manager(ca_cert_path=ca_cert_path)
@@ -364,14 +346,14 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL,
-                                         verify=ca_bundle.where())
+        get_mock.assert_called_once_with(REMOTE_KEY_URL, 'GET')
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_uses_combined_ca_bundle_otherwise(self):
         with tempfile.NamedTemporaryFile() as tmp_input, \
                 tempfile.NamedTemporaryFile(delete=False) as tmp_output:
-            ca_content = 'some\ncontent\n'
+            ca_content = pkg_resources.resource_string('leap.common.testing',
+                                                       'cacert.pem')
             ca_cert_path = tmp_input.name
             self._dump_to_file(ca_cert_path, ca_content)
 
@@ -383,8 +365,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
                 yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
                 # assert that combined bundle file is passed to get call
-                get_mock.assert_called_once_with(REMOTE_KEY_URL,
-                                                 verify=tmp_output.name)
+                get_mock.assert_called_once_with(REMOTE_KEY_URL, 'GET')
 
                 # assert that files got appended
                 expected = self._slurp_file(ca_bundle.where()) + ca_content
@@ -402,7 +383,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
             content = f.read()
         return content
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_decrypt_updates_sign_used_for_signer(self):
         # given
         km = self._key_manager()
@@ -420,7 +401,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         # then
         self.assertEqual(True, key.sign_used)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_decrypt_does_not_update_sign_used_for_recipient(self):
         # given
         km = self._key_manager()
@@ -445,7 +426,7 @@ class KeyManagerCryptoTestCase(KeyManagerWithSoledadTestCase):
 
     RAW_DATA = 'data'
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_keymanager_openpgp_encrypt_decrypt(self):
         km = self._key_manager()
         # put raw private key
@@ -464,7 +445,7 @@ class KeyManagerCryptoTestCase(KeyManagerWithSoledadTestCase):
                                fetch_remote=False)
         self.assertEqual(signingkey.fingerprint, key.fingerprint)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_keymanager_openpgp_encrypt_decrypt_wrong_sign(self):
         km = self._key_manager()
         # put raw keys
@@ -481,7 +462,7 @@ class KeyManagerCryptoTestCase(KeyManagerWithSoledadTestCase):
         self.assertEqual(self.RAW_DATA, rawdata)
         self.assertTrue(isinstance(signingkey, errors.InvalidSignature))
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_keymanager_openpgp_sign_verify(self):
         km = self._key_manager()
         # put raw private keys
-- 
cgit v1.2.3


From 00c0ed0d1b14aff8be0172e03bf26e1c30477af6 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 2 Feb 2016 19:51:36 -0300
Subject: [docs] add docstrings and fixes pep8

Some methods were missing docstrings and some code was exceeding the 80
column limit. Also some asserts arent needed anymore.
---
 src/leap/keymanager/__init__.py | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 1dcf642..7e4d30e 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -184,26 +184,29 @@ class KeyManager(object):
 
     def _key_class_from_type(self, ktype):
         """
-        Return key class from string representation of key type.
+        Given a class type, return a class
+
+        :param ktype: string representation of a class name
+        :type ktype: str
+
+        :return: A class with the matching name
+        :rtype: classobj or type
         """
         return filter(
             lambda klass: klass.__name__ == ktype,
             self._wrapper_map).pop()
 
     @defer.inlineCallbacks
-    def _get_json(self, address):
+    def _get_key_from_nicknym(self, address):
         """
         Send a GET request to C{uri} containing C{data}.
 
-        :param uri: The URI of the request.
-        :type uri: str
+        :param address: The URI of the request.
+        :type address: str
 
-        :return: A deferred that will be fired with the GET response
+        :return: A deferred that will be fired with GET content as json (dict)
         :rtype: Deferred
         """
-        leap_assert(
-            self._ca_cert_path is not None,
-            'We need the CA certificate path!')
         try:
             uri = self._nickserver_uri + '?address=' + address
             content = yield self._async_client_pinned.request(str(uri), 'GET')
@@ -214,7 +217,7 @@ class KeyManager(object):
                 # raise KeyNotFound(address)
             logger.warning("HTTP error retrieving key: %r" % (e,))
             logger.warning("%s" % (content,))
-            raise KeyNotFound(e), None, sys.exc_info()[2]
+            raise KeyNotFound(e.message), None, sys.exc_info()[2]
         except ValueError as v:
             logger.warning("Invalid JSON data from key: %s" % (uri,))
             raise KeyNotFound(v.message + ' - ' + uri), None, sys.exc_info()[2]
@@ -271,9 +274,6 @@ class KeyManager(object):
         :return: A deferred that will be fired when PUT request finishes
         :rtype: Deferred
         """
-        leap_assert(
-            self._ca_cert_path is not None,
-            'We need the CA certificate path!')
         leap_assert(
             self._token is not None,
             'We need a token to interact with webapp!')
@@ -282,7 +282,9 @@ class KeyManager(object):
         headers = {'Authorization': [str('Token token=%s' % self._token)]}
         headers['Content-Type'] = ['application/x-www-form-urlencoded']
         try:
-            res = yield self._async_client_pinned.request(str(uri), 'PUT', body=str(data), headers=headers)
+            res = yield self._async_client_pinned.request(str(uri), 'PUT',
+                                                          body=str(data),
+                                                          headers=headers)
         except Exception as e:
             logger.warning("Error uploading key: %r" % (e,))
             raise e
@@ -309,7 +311,7 @@ class KeyManager(object):
 
         """
         # request keys from the nickserver
-        server_keys = yield self._get_json(address)
+        server_keys = yield self._get_key_from_nicknym(address)
 
         # insert keys in local database
         if self.OPENPGP_KEY in server_keys:
@@ -357,7 +359,9 @@ class KeyManager(object):
                 self._api_version,
                 self._uid)
             d = self._put(uri, data)
-            emit_async(catalog.KEYMANAGER_DONE_UPLOADING_KEYS, self._address)
+            d.addCallback(lambda _:
+                          emit_async(catalog.KEYMANAGER_DONE_UPLOADING_KEYS,
+                                     self._address))
             return d
 
         d = self.get_key(
-- 
cgit v1.2.3


From 4e13b7595fc5c8e245244eb535525ae8333ef9dc Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Tue, 9 Feb 2016 16:32:53 +0100
Subject: [style] fix pep8

---
 src/leap/keymanager/tests/test_keymanager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index c3426b6..0f08326 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -253,7 +253,8 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         data = json.dumps({'address': address, 'openpgp': key})
 
         # mock the fetcher so it returns the key for ADDRESS_2
-        km._async_client_pinned.request = Mock(return_value=defer.succeed(data))
+        km._async_client_pinned.request = Mock(
+            return_value=defer.succeed(data))
         km.ca_cert_path = 'cacertpath'
         # try to key get without fetching from server
         d_fail = km.get_key(address, OpenPGPKey, fetch_remote=False)
-- 
cgit v1.2.3


From c18b69441e24c57a3130e27356edc0b9395d78f8 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Thu, 11 Feb 2016 19:22:34 -0300
Subject: [feat] defer decrypt, gen_key and encrypt

This commit put those gnupg operations to be run on external threads
limited by the amount of cores present on user machine.
Some gnupg calls spawn processes and communicating to them is a
synchronous operation, so running outside of a reactor should improve
response time by avoiding reactor locking.
---
 changes/async_gpg                         |  1 +
 src/leap/keymanager/__init__.py           | 18 +++++++-------
 src/leap/keymanager/openpgp.py            | 40 ++++++++++++++++++++++---------
 src/leap/keymanager/tests/test_openpgp.py | 30 +++++++++++------------
 4 files changed, 54 insertions(+), 35 deletions(-)
 create mode 100644 changes/async_gpg

diff --git a/changes/async_gpg b/changes/async_gpg
new file mode 100644
index 0000000..59d4d41
--- /dev/null
+++ b/changes/async_gpg
@@ -0,0 +1 @@
+-- Defer encrypt, decrypt and gen_key operations from gnupg to external threads, limited by cpu core amount.
diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 7e4d30e..9aa7139 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -572,15 +572,15 @@ class KeyManager(object):
         self._assert_supported_key_type(ktype)
         _keys = self._wrapper_map[ktype]
 
+        @defer.inlineCallbacks
         def encrypt(keys):
             pubkey, signkey = keys
-            encrypted = _keys.encrypt(
+            encrypted = yield _keys.encrypt(
                 data, pubkey, passphrase, sign=signkey,
                 cipher_algo=cipher_algo)
             pubkey.encr_used = True
-            d = _keys.put_key(pubkey, address)
-            d.addCallback(lambda _: encrypted)
-            return d
+            yield _keys.put_key(pubkey, address)
+            defer.returnValue(encrypted)
 
         dpub = self.get_key(address, ktype, private=False,
                             fetch_remote=fetch_remote)
@@ -625,9 +625,10 @@ class KeyManager(object):
         self._assert_supported_key_type(ktype)
         _keys = self._wrapper_map[ktype]
 
+        @defer.inlineCallbacks
         def decrypt(keys):
             pubkey, privkey = keys
-            decrypted, signed = _keys.decrypt(
+            decrypted, signed = yield _keys.decrypt(
                 data, privkey, passphrase=passphrase, verify=pubkey)
             if pubkey is None:
                 signature = KeyNotFound(verify)
@@ -635,14 +636,13 @@ class KeyManager(object):
                 signature = pubkey
                 if not pubkey.sign_used:
                     pubkey.sign_used = True
-                    d = _keys.put_key(pubkey, verify)
-                    d.addCallback(lambda _: (decrypted, signature))
-                    return d
+                    yield _keys.put_key(pubkey, verify)
+                    defer.returnValue((decrypted, signature))
             else:
                 signature = InvalidSignature(
                     'Failed to verify signature with key %s' %
                     (pubkey.key_id,))
-            return (decrypted, signature)
+            defer.returnValue((decrypted, signature))
 
         dpriv = self.get_key(address, ktype, private=True)
         dpub = defer.succeed(None)
diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index d648137..9064043 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -27,9 +27,11 @@ import io
 
 
 from datetime import datetime
+from multiprocessing import cpu_count
 from gnupg import GPG
 from gnupg.gnupg import GPGUtilities
 from twisted.internet import defer
+from twisted.internet.threads import deferToThread
 
 from leap.common.check import leap_assert, leap_assert_type, leap_check
 from leap.keymanager import errors
@@ -55,6 +57,16 @@ logger = logging.getLogger(__name__)
 # A temporary GPG keyring wrapped to provide OpenPGP functionality.
 #
 
+# This function will be used to call blocking GPG functions outside
+# of Twisted reactor and match the concurrent calls to the amount of CPU cores
+cpu_core_semaphore = defer.DeferredSemaphore(cpu_count())
+
+
+def from_thread(func, *args, **kwargs):
+    call = lambda: deferToThread(func, *args, **kwargs)
+    return cpu_core_semaphore.run(call)
+
+
 class TempGPGWrapper(object):
     """
     A context manager that wraps a temporary GPG keyring which only contains
@@ -253,6 +265,7 @@ class OpenPGPScheme(EncryptionScheme):
         # make sure the key does not already exist
         leap_assert(is_address(address), 'Not an user address: %s' % address)
 
+        @defer.inlineCallbacks
         def _gen_key(_):
             with TempGPGWrapper(gpgbinary=self._gpgbinary) as gpg:
                 # TODO: inspect result, or use decorator
@@ -264,7 +277,7 @@ class OpenPGPScheme(EncryptionScheme):
                     name_comment='')
                 logger.info("About to generate keys... "
                             "This might take SOME time.")
-                gpg.gen_key(params)
+                yield from_thread(gpg.gen_key, params)
                 logger.info("Keys for %s have been successfully "
                             "generated." % (address,))
                 pubkeys = gpg.list_keys()
@@ -293,7 +306,7 @@ class OpenPGPScheme(EncryptionScheme):
                         gpg.export_keys(key['fingerprint'], secret=secret))
                     d = self.put_key(openpgp_key, address)
                     deferreds.append(d)
-                return defer.gatherResults(deferreds)
+                yield defer.gatherResults(deferreds)
 
         def key_already_exists(_):
             raise errors.KeyAlreadyExists(address)
@@ -686,6 +699,7 @@ class OpenPGPScheme(EncryptionScheme):
             raise errors.GPGError(
                 'Failed to encrypt/decrypt: %s' % stderr)
 
+    @defer.inlineCallbacks
     def encrypt(self, data, pubkey, passphrase=None, sign=None,
                 cipher_algo='AES256'):
         """
@@ -700,8 +714,8 @@ class OpenPGPScheme(EncryptionScheme):
         :param cipher_algo: The cipher algorithm to use.
         :type cipher_algo: str
 
-        :return: The encrypted data.
-        :rtype: str
+        :return: A Deferred that will be fired with the encrypted data.
+        :rtype: defer.Deferred
 
         :raise EncryptError: Raised if failed encrypting for some reason.
         """
@@ -713,7 +727,8 @@ class OpenPGPScheme(EncryptionScheme):
             leap_assert(sign.private is True)
             keys.append(sign)
         with TempGPGWrapper(keys, self._gpgbinary) as gpg:
-            result = gpg.encrypt(
+            result = yield from_thread(
+                gpg.encrypt,
                 data, pubkey.fingerprint,
                 default_key=sign.key_id if sign else None,
                 passphrase=passphrase, symmetric=False,
@@ -724,11 +739,12 @@ class OpenPGPScheme(EncryptionScheme):
             # result.data  - (bool) contains the result of the operation
             try:
                 self._assert_gpg_result_ok(result)
-                return result.data
+                defer.returnValue(result.data)
             except errors.GPGError as e:
                 logger.error('Failed to decrypt: %s.' % str(e))
                 raise errors.EncryptError()
 
+    @defer.inlineCallbacks
     def decrypt(self, data, privkey, passphrase=None, verify=None):
         """
         Decrypt C{data} using private @{privkey} and verify with C{verify} key.
@@ -743,8 +759,9 @@ class OpenPGPScheme(EncryptionScheme):
         :param verify: The key used to verify a signature.
         :type verify: OpenPGPKey
 
-        :return: The decrypted data and if signature verifies
-        :rtype: (unicode, bool)
+        :return: Deferred that will fire with the decrypted data and
+                 if signature verifies (unicode, bool)
+        :rtype: Deferred
 
         :raise DecryptError: Raised if failed decrypting for some reason.
         """
@@ -756,8 +773,9 @@ class OpenPGPScheme(EncryptionScheme):
             keys.append(verify)
         with TempGPGWrapper(keys, self._gpgbinary) as gpg:
             try:
-                result = gpg.decrypt(
-                    data, passphrase=passphrase, always_trust=True)
+                result = yield from_thread(gpg.decrypt,
+                                           data, passphrase=passphrase,
+                                           always_trust=True)
                 self._assert_gpg_result_ok(result)
 
                 # verify signature
@@ -767,7 +785,7 @@ class OpenPGPScheme(EncryptionScheme):
                         verify.fingerprint == result.pubkey_fingerprint):
                     sign_valid = True
 
-                return (result.data, sign_valid)
+                defer.returnValue((result.data, sign_valid))
             except errors.GPGError as e:
                 logger.error('Failed to decrypt: %s.' % str(e))
                 raise errors.DecryptError(str(e))
diff --git a/src/leap/keymanager/tests/test_openpgp.py b/src/leap/keymanager/tests/test_openpgp.py
index bae83db..96b40a0 100644
--- a/src/leap/keymanager/tests/test_openpgp.py
+++ b/src/leap/keymanager/tests/test_openpgp.py
@@ -109,7 +109,7 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         # encrypt
         yield pgp.put_ascii_key(PUBLIC_KEY, ADDRESS)
         pubkey = yield pgp.get_key(ADDRESS, private=False)
-        cyphertext = pgp.encrypt(data, pubkey)
+        cyphertext = yield pgp.encrypt(data, pubkey)
 
         self.assertTrue(cyphertext is not None)
         self.assertTrue(cyphertext != '')
@@ -121,7 +121,7 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         yield self._assert_key_not_found(pgp, ADDRESS, private=True)
         yield pgp.put_ascii_key(PRIVATE_KEY, ADDRESS)
         privkey = yield pgp.get_key(ADDRESS, private=True)
-        decrypted, _ = pgp.decrypt(cyphertext, privkey)
+        decrypted, _ = yield pgp.decrypt(cyphertext, privkey)
         self.assertEqual(decrypted, data)
 
         yield pgp.delete_key(pubkey)
@@ -171,9 +171,9 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         yield pgp.put_ascii_key(PRIVATE_KEY, ADDRESS)
         privkey = yield pgp.get_key(ADDRESS, private=True)
         pubkey = yield pgp.get_key(ADDRESS, private=False)
-        self.assertRaises(
-            AssertionError,
-            pgp.encrypt, data, privkey, sign=pubkey)
+        self.failureResultOf(
+            pgp.encrypt(data, privkey, sign=pubkey),
+            AssertionError)
 
     @inlineCallbacks
     def test_decrypt_verify_with_private_raises(self):
@@ -183,12 +183,11 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         yield pgp.put_ascii_key(PRIVATE_KEY, ADDRESS)
         privkey = yield pgp.get_key(ADDRESS, private=True)
         pubkey = yield pgp.get_key(ADDRESS, private=False)
-        encrypted_and_signed = pgp.encrypt(
+        encrypted_and_signed = yield pgp.encrypt(
             data, pubkey, sign=privkey)
-        self.assertRaises(
-            AssertionError,
-            pgp.decrypt,
-            encrypted_and_signed, privkey, verify=privkey)
+        self.failureResultOf(
+            pgp.decrypt(encrypted_and_signed, privkey, verify=privkey),
+            AssertionError)
 
     @inlineCallbacks
     def test_decrypt_verify_with_wrong_key(self):
@@ -198,11 +197,12 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         yield pgp.put_ascii_key(PRIVATE_KEY, ADDRESS)
         privkey = yield pgp.get_key(ADDRESS, private=True)
         pubkey = yield pgp.get_key(ADDRESS, private=False)
-        encrypted_and_signed = pgp.encrypt(data, pubkey, sign=privkey)
+        encrypted_and_signed = yield pgp.encrypt(data, pubkey, sign=privkey)
         yield pgp.put_ascii_key(PUBLIC_KEY_2, ADDRESS_2)
         wrongkey = yield pgp.get_key(ADDRESS_2)
-        decrypted, validsign = pgp.decrypt(encrypted_and_signed, privkey,
-                                           verify=wrongkey)
+        decrypted, validsign = yield pgp.decrypt(encrypted_and_signed,
+                                                 privkey,
+                                                 verify=wrongkey)
         self.assertEqual(decrypted, data)
         self.assertFalse(validsign)
 
@@ -232,9 +232,9 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         privkey2 = yield pgp.get_key(ADDRESS_2, private=True)
 
         data = 'data'
-        encrypted_and_signed = pgp.encrypt(
+        encrypted_and_signed = yield pgp.encrypt(
             data, pubkey2, sign=privkey)
-        res, validsign = pgp.decrypt(
+        res, validsign = yield pgp.decrypt(
             encrypted_and_signed, privkey2, verify=pubkey)
         self.assertEqual(data, res)
         self.assertTrue(validsign)
-- 
cgit v1.2.3


From 64eaf2ee426623072bc2d9b1faf77ab831cb3be1 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Tue, 15 Dec 2015 13:29:44 +0100
Subject: [feat] move validation, usage and audited date to the active document

- Resolves: #7485
---
 changes/next-changelog.txt                   |  29 +++
 src/leap/keymanager/__init__.py              |   1 +
 src/leap/keymanager/keys.py                  | 132 ++++++++++---
 src/leap/keymanager/openpgp.py               | 268 +++++++++++----------------
 src/leap/keymanager/tests/test_keymanager.py |  15 +-
 src/leap/keymanager/tests/test_openpgp.py    |  54 +++++-
 src/leap/keymanager/tests/test_validation.py | 164 ++++++++++++++--
 7 files changed, 459 insertions(+), 204 deletions(-)
 create mode 100644 changes/next-changelog.txt

diff --git a/changes/next-changelog.txt b/changes/next-changelog.txt
new file mode 100644
index 0000000..c6cdde8
--- /dev/null
+++ b/changes/next-changelog.txt
@@ -0,0 +1,29 @@
+0.5.0 - xxx
++++++++++++++++++++++++++++++++
+
+Please add lines to this file, they will be moved to the CHANGELOG.rst during
+the next release.
+
+There are two template lines for each category, use them as reference.
+
+I've added a new category `Misc` so we can track doc/style/packaging stuff.
+
+Features
+~~~~~~~~
+- `#7485 <https://leap.se/code/issues/7485>`_: Move validation, usage and audited date to the active document.
+- `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
+- New feature without related issue number.
+
+Bugfixes
+~~~~~~~~
+- `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235.
+- Bugfix without related issue number.
+
+Misc
+~~~~
+- `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
+- Some change without issue number.
+
+Known Issues
+~~~~~~~~~~~~
+- `#1236 <https://leap.se/code/issues/1236>`_: Description of the known issue corresponding with issue #1236.
diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 9aa7139..8b3487f 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -439,6 +439,7 @@ class KeyManager(object):
         :return: A Deferred which fires with a list of all keys in local db.
         :rtype: Deferred
         """
+        # TODO: should it be based on activedocs?
         def build_keys(docs):
             return map(
                 lambda doc: build_key_from_dict(
diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index 91559c2..0de6a82 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -28,6 +28,7 @@ except ImportError:
 import logging
 import re
 import time
+import traceback
 
 
 from abc import ABCMeta, abstractmethod
@@ -110,39 +111,48 @@ def is_address(address):
     return bool(re.match('[\w.-]+@[\w.-]+', address))
 
 
-def build_key_from_dict(kClass, kdict):
+def build_key_from_dict(kClass, key, active=None):
     """
     Build an C{kClass} key based on info in C{kdict}.
 
-    :param kdict: Dictionary with key data.
-    :type kdict: dict
+    :param key: Dictionary with key data.
+    :type key: dict
+    :param active: Dictionary with active data.
+    :type active: dict
     :return: An instance of the key.
     :rtype: C{kClass}
     """
-    try:
-        validation = ValidationLevels.get(kdict[KEY_VALIDATION_KEY])
-    except ValueError:
-        logger.error("Not valid validation level (%s) for key %s",
-                     (kdict[KEY_VALIDATION_KEY], kdict[KEY_ID_KEY]))
-        validation = ValidationLevels.Weak_Chain
-
-    expiry_date = _to_datetime(kdict[KEY_EXPIRY_DATE_KEY])
-    last_audited_at = _to_datetime(kdict[KEY_LAST_AUDITED_AT_KEY])
-    refreshed_at = _to_datetime(kdict[KEY_REFRESHED_AT_KEY])
+    validation = ValidationLevels.Weak_Chain
+    last_audited_at = None
+    encr_used = False
+    sign_used = False
+
+    if active:
+        try:
+            validation = ValidationLevels.get(active[KEY_VALIDATION_KEY])
+        except ValueError:
+            logger.error("Not valid validation level (%s) for key %s",
+                         (active[KEY_VALIDATION_KEY], active[KEY_ID_KEY]))
+        last_audited_at = _to_datetime(active[KEY_LAST_AUDITED_AT_KEY])
+        encr_used = active[KEY_ENCR_USED_KEY]
+        sign_used = active[KEY_SIGN_USED_KEY]
+
+    expiry_date = _to_datetime(key[KEY_EXPIRY_DATE_KEY])
+    refreshed_at = _to_datetime(key[KEY_REFRESHED_AT_KEY])
 
     return kClass(
-        kdict[KEY_ADDRESS_KEY],
-        key_id=kdict[KEY_ID_KEY],
-        fingerprint=kdict[KEY_FINGERPRINT_KEY],
-        key_data=kdict[KEY_DATA_KEY],
-        private=kdict[KEY_PRIVATE_KEY],
-        length=kdict[KEY_LENGTH_KEY],
+        key[KEY_ADDRESS_KEY],
+        key_id=key[KEY_ID_KEY],
+        fingerprint=key[KEY_FINGERPRINT_KEY],
+        key_data=key[KEY_DATA_KEY],
+        private=key[KEY_PRIVATE_KEY],
+        length=key[KEY_LENGTH_KEY],
         expiry_date=expiry_date,
         last_audited_at=last_audited_at,
         refreshed_at=refreshed_at,
         validation=validation,
-        encr_used=kdict[KEY_ENCR_USED_KEY],
-        sign_used=kdict[KEY_SIGN_USED_KEY],
+        encr_used=encr_used,
+        sign_used=sign_used,
     )
 
 
@@ -178,6 +188,7 @@ class EncryptionKey(object):
                  key_data="", private=False, length=0, expiry_date=None,
                  validation=ValidationLevels.Weak_Chain, last_audited_at=None,
                  refreshed_at=None, encr_used=False, sign_used=False):
+        # TODO: it should know its own active address
         self.address = address
         self.key_id = key_id
         self.fingerprint = fingerprint
@@ -185,6 +196,7 @@ class EncryptionKey(object):
         self.private = private
         self.length = length
         self.expiry_date = expiry_date
+
         self.validation = validation
         self.last_audited_at = last_audited_at
         self.refreshed_at = refreshed_at
@@ -199,7 +211,6 @@ class EncryptionKey(object):
         :rtype: str
         """
         expiry_date = _to_unix_time(self.expiry_date)
-        last_audited_at = _to_unix_time(self.last_audited_at)
         refreshed_at = _to_unix_time(self.refreshed_at)
 
         return json.dumps({
@@ -211,11 +222,7 @@ class EncryptionKey(object):
             KEY_PRIVATE_KEY: self.private,
             KEY_LENGTH_KEY: self.length,
             KEY_EXPIRY_DATE_KEY: expiry_date,
-            KEY_LAST_AUDITED_AT_KEY: last_audited_at,
             KEY_REFRESHED_AT_KEY: refreshed_at,
-            KEY_VALIDATION_KEY: str(self.validation),
-            KEY_ENCR_USED_KEY: self.encr_used,
-            KEY_SIGN_USED_KEY: self.sign_used,
             KEY_TAGS_KEY: [KEYMANAGER_KEY_TAG],
         })
 
@@ -223,16 +230,20 @@ class EncryptionKey(object):
         """
         Return a JSON string describing this key.
 
-        :param address: Address for wich the key is active
-        :type address: str
         :return: The JSON string describing this key.
         :rtype: str
         """
+        last_audited_at = _to_unix_time(self.last_audited_at)
+
         return json.dumps({
             KEY_ADDRESS_KEY: address,
             KEY_TYPE_KEY: self.__class__.__name__ + KEYMANAGER_ACTIVE_TYPE,
             KEY_ID_KEY: self.key_id,
             KEY_PRIVATE_KEY: self.private,
+            KEY_VALIDATION_KEY: str(self.validation),
+            KEY_LAST_AUDITED_AT_KEY: last_audited_at,
+            KEY_ENCR_USED_KEY: self.encr_used,
+            KEY_SIGN_USED_KEY: self.sign_used,
             KEY_TAGS_KEY: [KEYMANAGER_ACTIVE_TAG],
         })
 
@@ -464,3 +475,68 @@ class EncryptionScheme(object):
         :rtype: bool
         """
         pass
+
+    def _repair_key_docs(self, doclist):
+        """
+        If there is more than one key for a key id try to self-repair it
+
+        :return: a Deferred that will be fired with the valid key doc once all
+                 the deletions are completed
+        :rtype: Deferred
+        """
+        def log_key_doc(doc):
+            logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
+                                       doc.content[KEY_FINGERPRINT_KEY]))
+
+        def cmp_key(d1, d2):
+            return cmp(d1.content[KEY_REFRESHED_AT_KEY],
+                       d2.content[KEY_REFRESHED_AT_KEY])
+
+        return self._repair_docs(doclist, cmp_key, log_key_doc)
+
+    def _repair_active_docs(self, doclist):
+        """
+        If there is more than one active doc for an address try to self-repair
+        it
+
+        :return: a Deferred that will be fired with the valid active doc once
+                 all the deletions are completed
+        :rtype: Deferred
+        """
+        def log_active_doc(doc):
+            logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
+                                       doc.content[KEY_ID_KEY]))
+
+        def cmp_active(d1, d2):
+            res = cmp(d1.content[KEY_LAST_AUDITED_AT_KEY],
+                      d2.content[KEY_LAST_AUDITED_AT_KEY])
+            if res != 0:
+                return res
+
+            used1 = (d1.content[KEY_SIGN_USED_KEY] +
+                     d1.content[KEY_ENCR_USED_KEY])
+            used2 = (d2.content[KEY_SIGN_USED_KEY] +
+                     d2.content[KEY_ENCR_USED_KEY])
+            return cmp(used1, used2)
+
+        return self._repair_docs(doclist, cmp_active, log_active_doc)
+
+    def _repair_docs(self, doclist, cmp_func, log_func):
+        logger.error("BUG ---------------------------------------------------")
+        logger.error("There is more than one doc of type %s:"
+                     % (doclist[0].content[KEY_TYPE_KEY],))
+
+        doclist.sort(cmp=cmp_func, reverse=True)
+        log_func(doclist[0])
+        deferreds = []
+        for doc in doclist[1:]:
+            log_func(doc)
+            d = self._soledad.delete_doc(doc)
+            deferreds.append(d)
+
+        logger.error("")
+        logger.error(traceback.extract_stack())
+        logger.error("BUG (please report above info) ------------------------")
+        d = defer.gatherResults(deferreds, consumeErrors=True)
+        d.addCallback(lambda _: doclist[0])
+        return d
diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index 9064043..3c8ac1e 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -22,7 +22,6 @@ import os
 import re
 import shutil
 import tempfile
-import traceback
 import io
 
 
@@ -44,8 +43,6 @@ from leap.keymanager.keys import (
     TYPE_ADDRESS_PRIVATE_INDEX,
     KEY_ADDRESS_KEY,
     KEY_ID_KEY,
-    KEY_FINGERPRINT_KEY,
-    KEY_REFRESHED_AT_KEY,
     KEYMANAGER_ACTIVE_TYPE,
 )
 
@@ -223,6 +220,41 @@ class OpenPGPKey(EncryptionKey):
 
         return []
 
+    def merge(self, newkey):
+        if newkey.fingerprint != self.fingerprint:
+            logger.critical(
+                "Can't put a key whith the same key_id and different "
+                "fingerprint: %s, %s"
+                % (newkey.fingerprint, self.fingerprint))
+            raise errors.KeyFingerprintMismatch(newkey.fingerprint)
+
+        with TempGPGWrapper(gpgbinary=self._gpgbinary) as gpg:
+            gpg.import_keys(self.key_data)
+            gpg.import_keys(newkey.key_data)
+            gpgkey = gpg.list_keys(secret=newkey.private).pop()
+
+            if gpgkey['expires']:
+                self.expiry_date = datetime.fromtimestamp(
+                    int(gpgkey['expires']))
+            else:
+                self.expiry_date = None
+
+            self.uids = []
+            for uid in gpgkey['uids']:
+                self.uids.append(_parse_address(uid))
+
+            self.length = int(gpgkey['length'])
+            self.key_data = gpg.export_keys(gpgkey['fingerprint'],
+                                            secret=self.private)
+
+        if newkey.validation > self.validation:
+            self.validation = newkey.validation
+        if newkey.last_audited_at > self.last_audited_at:
+            self.validation = newkey.last_audited_at
+        self.encr_used = newkey.encr_used or self.encr_used
+        self.sign_used = newkey.sign_used or self.sign_used
+        self.refreshed_at = datetime.now()
+
 
 class OpenPGPScheme(EncryptionScheme):
     """
@@ -332,15 +364,16 @@ class OpenPGPScheme(EncryptionScheme):
         """
         address = _parse_address(address)
 
-        def build_key(doc):
-            if doc is None:
+        def build_key((keydoc, activedoc)):
+            if keydoc is None:
                 raise errors.KeyNotFound(address)
             leap_assert(
-                address in doc.content[KEY_ADDRESS_KEY],
+                address in keydoc.content[KEY_ADDRESS_KEY],
                 'Wrong address in key %s. Expected %s, found %s.'
-                % (doc.content[KEY_ID_KEY], address,
-                   doc.content[KEY_ADDRESS_KEY]))
-            key = build_key_from_dict(OpenPGPKey, doc.content)
+                % (keydoc.content[KEY_ID_KEY], address,
+                   keydoc.content[KEY_ADDRESS_KEY]))
+            key = build_key_from_dict(OpenPGPKey, keydoc.content,
+                                      activedoc.content)
             key._gpgbinary = self._gpgbinary
             return key
 
@@ -426,100 +459,44 @@ class OpenPGPScheme(EncryptionScheme):
         :return: A Deferred which fires when the key is in the storage.
         :rtype: Deferred
         """
-        d = self._put_key_doc(key)
-        d.addCallback(lambda _: self._put_active_doc(key, address))
-        return d
+        def merge_and_put((keydoc, activedoc)):
+            if not keydoc:
+                return put_new_key(activedoc)
 
-    def _put_key_doc(self, key):
-        """
-        Put key document in soledad
+            active_content = None
+            if activedoc:
+                active_content = activedoc.content
+            oldkey = build_key_from_dict(OpenPGPKey, keydoc.content,
+                                         active_content)
 
-        :type key: OpenPGPKey
-        :rtype: Deferred
-        """
-        def check_and_put(docs, key):
-            deferred_repair = defer.succeed(None)
-            if len(docs) == 0:
-                return self._soledad.create_doc_from_json(key.get_json())
-            elif len(docs) > 1:
-                deferred_repair = self._repair_key_docs(docs, key.key_id)
-
-            doc = docs[0]
-            oldkey = build_key_from_dict(OpenPGPKey, doc.content)
-            if key.fingerprint != oldkey.fingerprint:
-                logger.critical(
-                    "Can't put a key whith the same key_id and different "
-                    "fingerprint: %s, %s"
-                    % (key.fingerprint, oldkey.fingerprint))
-                return defer.fail(
-                    errors.KeyFingerprintMismatch(key.fingerprint))
-
-            # in case of an update of the key merge them with gnupg
-            with TempGPGWrapper(gpgbinary=self._gpgbinary) as gpg:
-                gpg.import_keys(oldkey.key_data)
-                gpg.import_keys(key.key_data)
-                gpgkey = gpg.list_keys(secret=key.private).pop()
-                mergedkey = self._build_key_from_gpg(
-                    gpgkey,
-                    gpg.export_keys(gpgkey['fingerprint'],
-                                    secret=key.private))
-            mergedkey.validation = max(
-                [key.validation, oldkey.validation])
-            mergedkey.last_audited_at = oldkey.last_audited_at
-            mergedkey.refreshed_at = key.refreshed_at
-            mergedkey.encr_used = key.encr_used or oldkey.encr_used
-            mergedkey.sign_used = key.sign_used or oldkey.sign_used
-            doc.set_json(mergedkey.get_json())
-            deferred_put = self._soledad.put_doc(doc)
-
-            d = defer.gatherResults([deferred_put, deferred_repair])
-            d.addCallback(lambda res: res[0])
-            return d
+            key.merge(oldkey)
+            keydoc.set_json(key.get_json())
+            deferred_key = self._soledad.put_doc(keydoc)
 
-        d = self._soledad.get_from_index(
-            TYPE_ID_PRIVATE_INDEX,
-            self.KEY_TYPE,
-            key.key_id,
-            '1' if key.private else '0')
-        d.addCallback(check_and_put, key)
-        return d
+            active_json = key.get_active_json(address)
+            if activedoc:
+                activedoc.set_json(active_json)
+                deferred_active = self._soledad.put_doc(activedoc)
+            else:
+                deferred_active = self._soledad.create_doc_from_json(
+                    active_json)
 
-    def _put_active_doc(self, key, address):
-        """
-        Put active key document in soledad
+            return defer.gatherResults([deferred_key, deferred_active])
 
-        :type key: OpenPGPKey
-        :type addresses: str
-        :rtype: Deferred
-        """
-        def check_and_put(docs):
-            if len(docs) == 1:
-                doc = docs.pop()
-                doc.set_json(key.get_active_json(address))
-                d = self._soledad.put_doc(doc)
-            else:
-                if len(docs) > 1:
-                    logger.error("There is more than one active key document "
-                                 "for the address %s" % (address,))
-                    deferreds = []
-                    for doc in docs:
-                        delete = self._soledad.delete_doc(doc)
-                        deferreds.append(delete)
-                    d = defer.gatherResults(deferreds, consumeErrors=True)
-                else:
-                    d = defer.succeed(None)
-
-                d.addCallback(
-                    lambda _: self._soledad.create_doc_from_json(
-                        key.get_active_json(address)))
-            return d
+        def put_new_key(activedoc):
+            deferreds = []
+            if activedoc:
+                d = self._soledad.delete_doc(activedoc)
+                deferreds.append(d)
+            for json in [key.get_json(), key.get_active_json(address)]:
+                d = self._soledad.create_doc_from_json(json)
+                deferreds.append(d)
+            return defer.gatherResults(deferreds)
 
-        d = self._soledad.get_from_index(
-            TYPE_ADDRESS_PRIVATE_INDEX,
-            self.ACTIVE_TYPE,
-            address,
-            '1' if key.private else '0')
-        d.addCallback(check_and_put)
+        dk = self._get_key_doc_from_keyid(key.key_id, key.private)
+        da = self._get_active_doc_from_address(address, key.private)
+        d = defer.gatherResults([dk, da])
+        d.addCallback(merge_and_put)
         return d
 
     def _get_key_doc(self, address, private=False):
@@ -533,44 +510,26 @@ class OpenPGPScheme(EncryptionScheme):
         :param private: Whether to look for a private key.
         :type private: bool
 
-        :return: A Deferred which fires with the SoledadDocument with the key
-                 or None if it does not exist.
+        :return: A Deferred which fires with a touple of two SoledadDocument
+                 (keydoc, activedoc) or None if it does not exist.
         :rtype: Deferred
         """
         def get_key_from_active_doc(activedoc):
-            if len(activedoc) is 0:
-                return None
-            leap_assert(
-                len(activedoc) is 1,
-                'Found more than one key for address %s!' % (address,))
-
-            key_id = activedoc[0].content[KEY_ID_KEY]
-            d = self._soledad.get_from_index(
-                TYPE_ID_PRIVATE_INDEX,
-                self.KEY_TYPE,
-                key_id,
-                '1' if private else '0')
-            d.addCallback(get_doc, key_id, activedoc)
+            if not activedoc:
+                return (None, None)
+            key_id = activedoc.content[KEY_ID_KEY]
+            d = self._get_key_doc_from_keyid(key_id, private)
+            d.addCallback(delete_active_if_no_key, activedoc)
             return d
 
-        def get_doc(doclist, key_id, activedoc):
-            if len(doclist) == 0:
-                logger.warning('There is no key for id %s! Self-repairing it.'
-                               % (key_id))
+        def delete_active_if_no_key(keydoc, activedoc):
+            if not keydoc:
                 d = self._soledad.delete_doc(activedoc)
-                d.addCallback(lambda _: None)
-                return d
-            elif len(doclist) > 1:
-                d = self._repair_key_docs(doclist, key_id)
-                d.addCallback(lambda _: doclist[0])
+                d.addCallback(lambda _: (None, None))
                 return d
-            return doclist[0]
+            return (keydoc, activedoc)
 
-        d = self._soledad.get_from_index(
-            TYPE_ADDRESS_PRIVATE_INDEX,
-            self.ACTIVE_TYPE,
-            address,
-            '1' if private else '0')
+        d = self._get_active_doc_from_address(address, private)
         d.addCallback(get_key_from_active_doc)
         return d
 
@@ -647,36 +606,6 @@ class OpenPGPScheme(EncryptionScheme):
         d.addCallback(delete_key)
         return d
 
-    def _repair_key_docs(self, doclist, key_id):
-        """
-        If there is more than one key for a key id try to self-repair it
-
-        :return: a Deferred that will be fired once all the deletions are
-                 completed
-        :rtype: Deferred
-        """
-        logger.error("BUG ---------------------------------------------------")
-        logger.error("There is more than one key with the same key_id %s:"
-                     % (key_id,))
-
-        def log_key_doc(doc):
-            logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
-                                       doc.content[KEY_FINGERPRINT_KEY]))
-
-        doclist.sort(key=lambda doc: doc.content[KEY_REFRESHED_AT_KEY],
-                     reverse=True)
-        log_key_doc(doclist[0])
-        deferreds = []
-        for doc in doclist[1:]:
-            log_key_doc(doc)
-            d = self._soledad.delete_doc(doc)
-            deferreds.append(d)
-
-        logger.error("")
-        logger.error(traceback.extract_stack())
-        logger.error("BUG (please report above info) ------------------------")
-        return defer.gatherResults(deferreds, consumeErrors=True)
-
     #
     # Data encryption, decryption, signing and verifying
     #
@@ -885,6 +814,31 @@ class OpenPGPScheme(EncryptionScheme):
             kfprint = gpgpubkey['fingerprint']
             return valid and rfprint == kfprint
 
+    def _get_active_doc_from_address(self, address, private):
+        d = self._soledad.get_from_index(
+            TYPE_ADDRESS_PRIVATE_INDEX,
+            self.ACTIVE_TYPE,
+            address,
+            '1' if private else '0')
+        d.addCallback(self._repair_and_get_doc, self._repair_active_docs)
+        return d
+
+    def _get_key_doc_from_keyid(self, key_id, private):
+        d = self._soledad.get_from_index(
+            TYPE_ID_PRIVATE_INDEX,
+            self.KEY_TYPE,
+            key_id,
+            '1' if private else '0')
+        d.addCallback(self._repair_and_get_doc, self._repair_key_docs)
+        return d
+
+    def _repair_and_get_doc(self, doclist, repair_func):
+        if len(doclist) is 0:
+            return None
+        elif len(doclist) > 1:
+            return repair_func(doclist)
+        return doclist[0]
+
 
 def process_ascii_key(key_data, gpgbinary, secret=False):
     with TempGPGWrapper(gpgbinary=gpgbinary) as gpg:
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index 0f08326..e4e0d8b 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -83,13 +83,18 @@ class KeyManagerUtilTestCase(unittest.TestCase):
             'private': False,
             'length': 4096,
             'expiry_date': 0,
-            'last_audited_at': 0,
             'refreshed_at': 1311239602,
+        }
+        adict = {
+            'address': ADDRESS,
+            'key_id': KEY_FINGERPRINT[-16:],
+            'private': False,
+            'last_audited_at': 0,
             'validation': str(ValidationLevels.Weak_Chain),
             'encr_used': False,
             'sign_used': True,
         }
-        key = build_key_from_dict(OpenPGPKey, kdict)
+        key = build_key_from_dict(OpenPGPKey, kdict, adict)
         self.assertEqual(
             kdict['address'], key.address,
             'Wrong data in key.')
@@ -118,13 +123,13 @@ class KeyManagerUtilTestCase(unittest.TestCase):
             datetime.fromtimestamp(kdict['refreshed_at']), key.refreshed_at,
             'Wrong data in key.')
         self.assertEqual(
-            ValidationLevels.get(kdict['validation']), key.validation,
+            ValidationLevels.get(adict['validation']), key.validation,
             'Wrong data in key.')
         self.assertEqual(
-            kdict['encr_used'], key.encr_used,
+            adict['encr_used'], key.encr_used,
             'Wrong data in key.')
         self.assertEqual(
-            kdict['sign_used'], key.sign_used,
+            adict['sign_used'], key.sign_used,
             'Wrong data in key.')
 
 
diff --git a/src/leap/keymanager/tests/test_openpgp.py b/src/leap/keymanager/tests/test_openpgp.py
index 96b40a0..6641591 100644
--- a/src/leap/keymanager/tests/test_openpgp.py
+++ b/src/leap/keymanager/tests/test_openpgp.py
@@ -29,7 +29,10 @@ from leap.keymanager import (
     KeyNotFound,
     openpgp,
 )
-from leap.keymanager.keys import TYPE_ID_PRIVATE_INDEX
+from leap.keymanager.keys import (
+    TYPE_ID_PRIVATE_INDEX,
+    TYPE_ADDRESS_PRIVATE_INDEX,
+)
 from leap.keymanager.openpgp import OpenPGPKey
 from leap.keymanager.tests import (
     KeyManagerWithSoledadTestCase,
@@ -308,6 +311,7 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         try:
             yield self.assertFailure(pgp.get_key(ADDRESS, private=False),
                                      KeyNotFound)
+            # it should have deleted the index
             self.assertEqual(self._soledad.delete_doc.call_count, 1)
         finally:
             self._soledad.get_from_index = get_from_index
@@ -349,6 +353,54 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
             self._soledad.get_from_index = get_from_index
             self._soledad.delete_doc = delete_doc
 
+    @inlineCallbacks
+    def test_self_repair_five_active_docs(self):
+        pgp = openpgp.OpenPGPScheme(
+            self._soledad, gpgbinary=self.gpg_binary_path)
+
+        get_from_index = self._soledad.get_from_index
+        delete_doc = self._soledad.delete_doc
+
+        def my_get_from_index(*args):
+            if (args[0] == TYPE_ADDRESS_PRIVATE_INDEX and
+                    args[2] == ADDRESS):
+                k1 = OpenPGPKey(ADDRESS, key_id="1",
+                                last_audited_at=datetime(2005, 1, 1))
+                k2 = OpenPGPKey(ADDRESS, key_id="2",
+                                last_audited_at=datetime(2007, 1, 1))
+                k3 = OpenPGPKey(ADDRESS, key_id="3",
+                                last_audited_at=datetime(2007, 1, 1),
+                                encr_used=True, sign_used=True)
+                k4 = OpenPGPKey(ADDRESS, key_id="4",
+                                last_audited_at=datetime(2007, 1, 1),
+                                sign_used=True)
+                k5 = OpenPGPKey(ADDRESS, key_id="5",
+                                last_audited_at=datetime(2007, 1, 1),
+                                encr_used=True)
+                deferreds = []
+                for k in [k1, k2, k3, k4, k5]:
+                    d = self._soledad.create_doc_from_json(
+                        k.get_active_json(ADDRESS))
+                    deferreds.append(d)
+                return gatherResults(deferreds)
+            elif args[0] == TYPE_ID_PRIVATE_INDEX:
+                key_id = args[2]
+                self.assertEqual(key_id, "3")
+                k = OpenPGPKey(ADDRESS, key_id="3")
+                return succeed(
+                    [self._soledad.create_doc_from_json(k.get_json())])
+            return get_from_index(*args)
+
+        self._soledad.get_from_index = my_get_from_index
+        self._soledad.delete_doc = Mock(return_value=succeed(None))
+
+        try:
+            yield pgp.get_key(ADDRESS, private=False)
+            self.assertEqual(self._soledad.delete_doc.call_count, 4)
+        finally:
+            self._soledad.get_from_index = get_from_index
+            self._soledad.delete_doc = delete_doc
+
     def _assert_key_not_found(self, pgp, address, private=False):
         d = pgp.get_key(address, private=private)
         return self.assertFailure(d, KeyNotFound)
diff --git a/src/leap/keymanager/tests/test_validation.py b/src/leap/keymanager/tests/test_validation.py
index bcf41c4..44d6e39 100644
--- a/src/leap/keymanager/tests/test_validation.py
+++ b/src/leap/keymanager/tests/test_validation.py
@@ -108,14 +108,14 @@ class ValidationLevelsTestCase(KeyManagerWithSoledadTestCase):
         TEXT = "some text"
 
         km = self._key_manager()
+        yield km.put_raw_key(UNEXPIRED_PRIVATE, OpenPGPKey, ADDRESS)
+        signature = yield km.sign(TEXT, ADDRESS, OpenPGPKey)
+        yield self.delete_all_keys(km)
+
         yield km.put_raw_key(UNEXPIRED_KEY, OpenPGPKey, ADDRESS)
         yield km.encrypt(TEXT, ADDRESS, OpenPGPKey)
-
-        km2 = self._key_manager()
-        yield km2.put_raw_key(UNEXPIRED_PRIVATE, OpenPGPKey, ADDRESS)
-        signature = yield km2.sign(TEXT, ADDRESS, OpenPGPKey)
-
         yield km.verify(TEXT, ADDRESS, OpenPGPKey, detached_sig=signature)
+
         d = km.put_raw_key(
             UNRELATED_KEY, OpenPGPKey, ADDRESS,
             validation=ValidationLevels.Provider_Endorsement)
@@ -126,17 +126,17 @@ class ValidationLevelsTestCase(KeyManagerWithSoledadTestCase):
         TEXT = "some text"
 
         km = self._key_manager()
+        yield km.put_raw_key(UNEXPIRED_PRIVATE, OpenPGPKey, ADDRESS)
+        yield km.put_raw_key(PUBLIC_KEY_2, OpenPGPKey, ADDRESS_2)
+        encrypted = yield km.encrypt(TEXT, ADDRESS_2, OpenPGPKey,
+                                     sign=ADDRESS)
+        yield self.delete_all_keys(km)
+
         yield km.put_raw_key(UNEXPIRED_KEY, OpenPGPKey, ADDRESS)
         yield km.put_raw_key(PRIVATE_KEY_2, OpenPGPKey, ADDRESS_2)
         yield km.encrypt(TEXT, ADDRESS, OpenPGPKey)
-
-        km2 = self._key_manager()
-        yield km2.put_raw_key(UNEXPIRED_PRIVATE, OpenPGPKey, ADDRESS)
-        yield km2.put_raw_key(PUBLIC_KEY_2, OpenPGPKey, ADDRESS_2)
-        encrypted = yield km2.encrypt(TEXT, ADDRESS_2, OpenPGPKey,
-                                      sign=ADDRESS)
-
         yield km.decrypt(encrypted, ADDRESS_2, OpenPGPKey, verify=ADDRESS)
+
         d = km.put_raw_key(
             UNRELATED_KEY, OpenPGPKey, ADDRESS,
             validation=ValidationLevels.Provider_Endorsement)
@@ -150,6 +150,33 @@ class ValidationLevelsTestCase(KeyManagerWithSoledadTestCase):
         key = yield km.get_key(ADDRESS, OpenPGPKey, fetch_remote=False)
         self.assertEqual(key.fingerprint, SIGNED_FINGERPRINT)
 
+    @inlineCallbacks
+    def test_two_uuids(self):
+        TEXT = "some text"
+
+        km = self._key_manager()
+        yield km.put_raw_key(UUIDS_PRIVATE, OpenPGPKey, ADDRESS_2)
+        signature = yield km.sign(TEXT, ADDRESS_2, OpenPGPKey)
+        yield self.delete_all_keys(km)
+
+        yield km.put_raw_key(UUIDS_KEY, OpenPGPKey, ADDRESS_2)
+        yield km.put_raw_key(UUIDS_KEY, OpenPGPKey, ADDRESS)
+        yield km.encrypt(TEXT, ADDRESS_2, OpenPGPKey)
+        yield km.verify(TEXT, ADDRESS_2, OpenPGPKey, detached_sig=signature)
+
+        d = km.put_raw_key(
+            PUBLIC_KEY_2, OpenPGPKey, ADDRESS_2,
+            validation=ValidationLevels.Provider_Endorsement)
+        yield self.assertFailure(d, KeyNotValidUpgrade)
+        key = yield km.get_key(ADDRESS_2, OpenPGPKey, fetch_remote=False)
+        self.assertEqual(key.fingerprint, UUIDS_FINGERPRINT)
+
+        yield km.put_raw_key(
+            PUBLIC_KEY, OpenPGPKey, ADDRESS,
+            validation=ValidationLevels.Provider_Endorsement)
+        key = yield km.get_key(ADDRESS, OpenPGPKey, fetch_remote=False)
+        self.assertEqual(key.fingerprint, KEY_FINGERPRINT)
+
 
 # Key material for testing
 
@@ -364,6 +391,117 @@ X2+l7IOSt+31KQCBFN/VmhTySJOVQC1d2A56lSH2c/DWVClji+x3suzn
 -----END PGP PUBLIC KEY BLOCK-----
 """
 
+# key 0x1DDBAEB928D982F7: public key two uuids
+#    uid                  anotheruser <anotheruser@leap.se>
+#    uid                  Leap Test Key <leap@leap.se>
+UUIDS_FINGERPRINT = "21690ED054C1B2F3ACE963D38FCC7DEFB4EE5A9B"
+UUIDS_KEY = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQENBFZwjz0BCADHpVg7js8PK9gQXx3Z+Jtj6gswYZpeXRdIUfZBSebWNGKwXxC9
+ZZDjnQc3l6Kezh7ra/hB6xulDbj3vXi66V279QSOuFAKMIudlJehb86bUiVk9Ppy
+kdrn44P40ZdVmw2m61WrKnvBelKW7pIF/EB/dY1uUonSfR56f/BxL5a5jGi2uI2y
+2hTPnZEksoKQsjsp1FckPFwVGzRKVzYl3lsorL5oiHd450G2B3VRw8AZ8Eqq6bzf
+jNrrA3TOMmEIYdADPqqnBj+xbnOCGBsvx+iAhGRxUckVJqW92SXlNNds8kEyoE0t
+9B6eqe0MrrlcK3SLe03j85T9f1o3An53rV/3ABEBAAG0HExlYXAgVGVzdCBLZXkg
+PGxlYXBAbGVhcC5zZT6JAT0EEwEIACcFAlZwjz0CGwMFCRLMAwAFCwkIBwMFFQoJ
+CAsFFgMCAQACHgECF4AACgkQj8x977TuWpu4ZggAgk6rVtOCqYwM720Bs3k+wsWu
+IVPUqnlPbSWph/PKBKWYE/5HoIGdvfN9jJxwpCM5x/ivPe1zeJ0qa9SnO66kolHU
+7qC8HRWC67R4znO4Zrs2I+SgwRHAPPbqMVPsNs5rS0D6DCcr+LXtJF+LLAsIwDfw
+mXEsKbX5H5aBmmDnfq0pGx05E3tKs5l09VVESvVZYOCM9b4FtdLVvgbKAD+KYDW1
+5A/PzOvyYjZu2FGtPKmNmqHD3KW8cmrcI/7mRR08FnNGbbpgXPZ2GPKgkUllY9N7
+E9lg4eaYH2fIWun293kbqp8ueELZvoU1jUQrP5B+eqBWTvIucqdQqJaiWn9pELQh
+YW5vdGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iQE9BBMBCAAnBQJWcI9a
+AhsDBQkSzAMABQsJCAcDBRUKCQgLBRYDAgEAAh4BAheAAAoJEI/Mfe+07lqblRMH
+/17vK2WOd0F7EegA5ELOrM+QJPKpLK4e6WdROUpS1hvRQcAOqadCCpYPSTTG/HnO
+d9/Q9Q/G3xHHlk6nl1qHRkVlp0iVWyBZFn1s8lgGz/FFUEXXRj7I5RGmKSNgDlqA
+un2OrwB2m+DH6pMjizu/RUfIJM2bSgViuvjCQoaLYLcFiULJlWHb/2WgpvesFyAc
+0oc9AkXuaCEo50XQlrt8Bh++6rfbAMAS7ZrHluFPIY+y4eVl+MU/QkoGYAVgiLLV
+5tnvbDZWNs8ixw4ubqKXYj5mK55sapokhOqObEfY6D3p7YpdQO/IhBneCw9gKOxa
+wYAPhCOrJC8JmE69I1Nk8Bu5AQ0EVnCPPQEIANUivsrR2jwb8C9wPONn0HS3YYCI
+/IVlLdw/Ia23ogBF1Uh8ORNg1G/t0/6S7IKDZ2gGgWw25u9TjWRRWsxO9tjOPi2d
+YuhwlQRnq5Or+LzIEKRs9GnJMLFT0kR9Nrhw4UyaN6tWkR9p1Py7ux8RLmDEMAs3
+fBfVMLMmQRerJ5SyCUiq/lm9aYTLCC+vkjE01C+2oI0gcWGfLDjiJbaD4AazzibP
+fBe41BIk7WaCJyEcBqfrgW/Seof43FhSKRGgc5nx3HH1DMz9AtYfKnVS5DgoBGpO
+hxgzIJN3/hFHPicRlYoHxLUE48GcFfQFEJ78RXeBuieXAkDxNczHnLkEPx8AEQEA
+AYkBJQQYAQgADwUCVnCPPQIbDAUJEswDAAAKCRCPzH3vtO5amyRIB/9IsWUWrvbH
+njw2ZCiIV++lHgIntAcuQIbZIjyMXoM+imHsPrsDOUT65yi9Xp1pUyZEKtGkZvP4
+p7HRzJL+RWiWEN7sgQfNqqev8BF2/xmxkmiSuXHJ0PSaC5DmLfFDyvvSU6H5VPud
+NszKIHtyoS6ph6TH8GXzFmNiHaTOZKdmVxlyLj1/DN+d63M+ARZIe7gB6jmP/gg4
+o52icfTcqLkkppYn8g1A9bdJ3k8qqExNPTf2llDscuD9VzpebFbPqfcYqR71GfG7
+Kmg7qGnZtNac1ERvknI/fmmCQydGk5pOh0KUTgeLG2qB07cqCUBbOXaweNWbiM9E
+vtQLNMD9Gn7D
+=MCXv
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+UUIDS_PRIVATE = """
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1
+
+lQOYBFZwjz0BCADHpVg7js8PK9gQXx3Z+Jtj6gswYZpeXRdIUfZBSebWNGKwXxC9
+ZZDjnQc3l6Kezh7ra/hB6xulDbj3vXi66V279QSOuFAKMIudlJehb86bUiVk9Ppy
+kdrn44P40ZdVmw2m61WrKnvBelKW7pIF/EB/dY1uUonSfR56f/BxL5a5jGi2uI2y
+2hTPnZEksoKQsjsp1FckPFwVGzRKVzYl3lsorL5oiHd450G2B3VRw8AZ8Eqq6bzf
+jNrrA3TOMmEIYdADPqqnBj+xbnOCGBsvx+iAhGRxUckVJqW92SXlNNds8kEyoE0t
+9B6eqe0MrrlcK3SLe03j85T9f1o3An53rV/3ABEBAAEAB/9Lzeg2lP7hz8/2R2da
+QB8gTNl6wVSPx+DzQMuz9o+DfdiLB02f3FSrWBBJd3XzvmfXE+Prg423mgJFbtfM
+gJdqqpnUZv9dHxmj96urTHyyVPqF3s7JecAYlDaj31EK3BjO7ERW/YaH7B432NXx
+F9qVitjsrsJN/dv4v2NYVq1wPcdDB05ge9WP+KRec7xvdTudH4Kov0iMZ+1Nksfn
+lrowGuMYBGWDlTNoBoEYxDD2lqVaiOjyjx5Ss8btS59IlXxApOFZTezekl7hUI2B
+1fDQ1GELL6BKVKxApGSD5XAgVlqkek4RhoHmg4gKSymfbFV5oRp1v1kC0JIGvnB1
+W5/BBADKzagL4JRnhWGubLec917B3se/3y1aHrEmO3T0IzxnUMD5yvg81uJWi5Co
+M05Nu/Ny/Fw1VgoF8MjiGnumB2KKytylu8LKLarDxPpLxabOBCQLHrLQOMsmesjR
+Cg3iYl/EeM/ooAufaN4IObcu6Pa8//rwNE7Fz1ZsIyJefN4fnwQA/AOpqA2BvHoH
+VRYh4NVuMLhF1YdKqcd/T/dtSqszcadkmG4+vAL76r3jYxScRoNGQaIkpBnzP0ry
+Adb0NDuBgSe/Cn44kqT7JJxTMfJNrw2rBMMUYZHdQrck2pf5R4fZ74yJyvCKg5pQ
+QAl1gTSi6PJvPwpc7m7Kr4kHBVDlgKkEAJKkVrtq/v2+jSA+/osa4YC5brsDQ08X
+pvZf0MBkc5/GDfusHyE8HGFnVY5ycrS85TBCrhc7suFu59pF4wEeXsqxqNf02gRe
+B+btPwR7yki73iyXs4cbuszHMD03UnbvItFAybD5CC+oR9kG5noI0TzJNUNX9Vkq
+xATf819dhwtgTha0HExlYXAgVGVzdCBLZXkgPGxlYXBAbGVhcC5zZT6JAT0EEwEI
+ACcFAlZwjz0CGwMFCRLMAwAFCwkIBwMFFQoJCAsFFgMCAQACHgECF4AACgkQj8x9
+77TuWpu4ZggAgk6rVtOCqYwM720Bs3k+wsWuIVPUqnlPbSWph/PKBKWYE/5HoIGd
+vfN9jJxwpCM5x/ivPe1zeJ0qa9SnO66kolHU7qC8HRWC67R4znO4Zrs2I+SgwRHA
+PPbqMVPsNs5rS0D6DCcr+LXtJF+LLAsIwDfwmXEsKbX5H5aBmmDnfq0pGx05E3tK
+s5l09VVESvVZYOCM9b4FtdLVvgbKAD+KYDW15A/PzOvyYjZu2FGtPKmNmqHD3KW8
+cmrcI/7mRR08FnNGbbpgXPZ2GPKgkUllY9N7E9lg4eaYH2fIWun293kbqp8ueELZ
+voU1jUQrP5B+eqBWTvIucqdQqJaiWn9pELQhYW5vdGhlcnVzZXIgPGFub3RoZXJ1
+c2VyQGxlYXAuc2U+iQE9BBMBCAAnBQJWcI9aAhsDBQkSzAMABQsJCAcDBRUKCQgL
+BRYDAgEAAh4BAheAAAoJEI/Mfe+07lqblRMH/17vK2WOd0F7EegA5ELOrM+QJPKp
+LK4e6WdROUpS1hvRQcAOqadCCpYPSTTG/HnOd9/Q9Q/G3xHHlk6nl1qHRkVlp0iV
+WyBZFn1s8lgGz/FFUEXXRj7I5RGmKSNgDlqAun2OrwB2m+DH6pMjizu/RUfIJM2b
+SgViuvjCQoaLYLcFiULJlWHb/2WgpvesFyAc0oc9AkXuaCEo50XQlrt8Bh++6rfb
+AMAS7ZrHluFPIY+y4eVl+MU/QkoGYAVgiLLV5tnvbDZWNs8ixw4ubqKXYj5mK55s
+apokhOqObEfY6D3p7YpdQO/IhBneCw9gKOxawYAPhCOrJC8JmE69I1Nk8BudA5gE
+VnCPPQEIANUivsrR2jwb8C9wPONn0HS3YYCI/IVlLdw/Ia23ogBF1Uh8ORNg1G/t
+0/6S7IKDZ2gGgWw25u9TjWRRWsxO9tjOPi2dYuhwlQRnq5Or+LzIEKRs9GnJMLFT
+0kR9Nrhw4UyaN6tWkR9p1Py7ux8RLmDEMAs3fBfVMLMmQRerJ5SyCUiq/lm9aYTL
+CC+vkjE01C+2oI0gcWGfLDjiJbaD4AazzibPfBe41BIk7WaCJyEcBqfrgW/Seof4
+3FhSKRGgc5nx3HH1DMz9AtYfKnVS5DgoBGpOhxgzIJN3/hFHPicRlYoHxLUE48Gc
+FfQFEJ78RXeBuieXAkDxNczHnLkEPx8AEQEAAQAH+wRSCn0RCPP7+v/zLgDMG3Eq
+QHs7C6dmmCnlS7j6Rnnr8HliL0QBy/yi3Q/Fia7RnBiDPT9k04SZdH3KmmUW2rEl
+aSRCkv00PwkSUuuQ6l9lTNUQclnsnqSRlusVgLT3cNG9NJCwFgwFeLBQ2+ey0PZc
+M78edlEDXNPc3CfvK8O7WK74YiNJqIQCs7aDJSv0s7O/asRQyMCsl/UYtMV6W03d
+eauS3bM41ll7GVfHMgkChFUQHb+19JHzSq4yeqQ/vS30ASugFxB3Omomp95sRL/w
+60y51faLyTKD4AN3FhDfeIEfh1ggN2UT70qzC3+F8TvxQQHEBhNQKlhWVbWTp+0E
+ANkcyokvn+09OIO/YDxF3aB37gA3J37d6NXos6pdvqUPOVSHvmF/hiM0FO2To6vu
+ex/WcDQafPm4eiW6oNllwtZhWU2tr34bZD4PIuktSX2Ax2v5QtZ4d1CVdDEwbYn/
+fmR+nif1fTKTljZragaI9Rt6NWhfh7UGt62iIKh0lbhLBAD7T5wHY8V1yqlnyByG
+K7nt+IHnND2I7Hk58yxKjv2KUNYxWZeOAQTQmfQXjJ+BOmw6oHMmDmGvdjSxIE+9
+j9nezEONxzVVjEDTKBeEnUeDY1QGDyDyW1/AhLJ52yWGTNrmKcGV4KmaYnhDzq7Z
+aqJVRcFMF9TAfhrEGGhRdD83/QQA6xAqjWiu6tbaDurVuce64mA1R3T7HJ81gEaX
+I+eJNDJb7PK3dhFETgyc3mcepWFNJkoXqx2ADhG8jLqK4o/N/QlV5PQgeHmhz09V
+Z7MNhivGxDKZoxX6Bouh+qs5OkatcGFhTz//+FHSfusV2emxNiwd4QIsizxaltqh
+W1ke0bA7eYkBJQQYAQgADwUCVnCPPQIbDAUJEswDAAAKCRCPzH3vtO5amyRIB/9I
+sWUWrvbHnjw2ZCiIV++lHgIntAcuQIbZIjyMXoM+imHsPrsDOUT65yi9Xp1pUyZE
+KtGkZvP4p7HRzJL+RWiWEN7sgQfNqqev8BF2/xmxkmiSuXHJ0PSaC5DmLfFDyvvS
+U6H5VPudNszKIHtyoS6ph6TH8GXzFmNiHaTOZKdmVxlyLj1/DN+d63M+ARZIe7gB
+6jmP/gg4o52icfTcqLkkppYn8g1A9bdJ3k8qqExNPTf2llDscuD9VzpebFbPqfcY
+qR71GfG7Kmg7qGnZtNac1ERvknI/fmmCQydGk5pOh0KUTgeLG2qB07cqCUBbOXaw
+eNWbiM9EvtQLNMD9Gn7D
+=/3u/
+-----END PGP PRIVATE KEY BLOCK-----
+"""
+
 if __name__ == "__main__":
-    import unittest
     unittest.main()
-- 
cgit v1.2.3


From 81232da09286f7f1812f6d3d182cd57665feaa1f Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Fri, 18 Dec 2015 19:37:44 +0100
Subject: [feat] Migrate soledad documents by adding versioning field

- Resolves: #7713
---
 changes/next-changelog.txt            |   2 +
 src/leap/keymanager/keys.py           |  22 ++++-
 src/leap/keymanager/migrator.py       | 171 ++++++++++++++++++++++++++++++++++
 src/leap/keymanager/tests/__init__.py |   2 +-
 4 files changed, 192 insertions(+), 5 deletions(-)
 create mode 100644 src/leap/keymanager/migrator.py

diff --git a/changes/next-changelog.txt b/changes/next-changelog.txt
index c6cdde8..de837da 100644
--- a/changes/next-changelog.txt
+++ b/changes/next-changelog.txt
@@ -11,6 +11,8 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff.
 Features
 ~~~~~~~~
 - `#7485 <https://leap.se/code/issues/7485>`_: Move validation, usage and audited date to the active document.
+- `#7713 <https://leap.se/code/issues/7713>`_: Update soledad documents by adding versioning field.
+
 - `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
 - New feature without related issue number.
 
diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index 0de6a82..a60c19d 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -45,6 +45,7 @@ logger = logging.getLogger(__name__)
 # Dictionary keys used for storing cryptographic keys.
 #
 
+KEY_VERSION_KEY = 'version'
 KEY_ADDRESS_KEY = 'address'
 KEY_TYPE_KEY = 'type'
 KEY_ID_KEY = 'key_id'
@@ -69,6 +70,10 @@ KEYMANAGER_KEY_TAG = 'keymanager-key'
 KEYMANAGER_ACTIVE_TAG = 'keymanager-active'
 KEYMANAGER_ACTIVE_TYPE = '-active'
 
+# Version of the Soledad Document schema,
+# it should be bumped each time the document format changes
+KEYMANAGER_DOC_VERSION = 1
+
 
 #
 # key indexing constants.
@@ -223,6 +228,7 @@ class EncryptionKey(object):
             KEY_LENGTH_KEY: self.length,
             KEY_EXPIRY_DATE_KEY: expiry_date,
             KEY_REFRESHED_AT_KEY: refreshed_at,
+            KEY_VERSION_KEY: KEYMANAGER_DOC_VERSION,
             KEY_TAGS_KEY: [KEYMANAGER_KEY_TAG],
         })
 
@@ -244,6 +250,7 @@ class EncryptionKey(object):
             KEY_LAST_AUDITED_AT_KEY: last_audited_at,
             KEY_ENCR_USED_KEY: self.encr_used,
             KEY_SIGN_USED_KEY: self.sign_used,
+            KEY_VERSION_KEY: KEYMANAGER_DOC_VERSION,
             KEY_TAGS_KEY: [KEYMANAGER_ACTIVE_TAG],
         })
 
@@ -281,7 +288,8 @@ class EncryptionScheme(object):
         :type soledad: leap.soledad.Soledad
         """
         self._soledad = soledad
-        self._init_indexes()
+        self.deferred_init = self._init_indexes()
+        self.deferred_init.addCallback(self._migrate_documents_schema)
 
     def _init_indexes(self):
         """
@@ -309,8 +317,14 @@ class EncryptionScheme(object):
                     deferreds.append(d)
             return defer.gatherResults(deferreds, consumeErrors=True)
 
-        self.deferred_indexes = self._soledad.list_indexes()
-        self.deferred_indexes.addCallback(init_idexes)
+        d = self._soledad.list_indexes()
+        d.addCallback(init_idexes)
+        return d
+
+    def _migrate_documents_schema(self, _):
+        from leap.keymanager.migrator import KeyDocumentsMigrator
+        migrator = KeyDocumentsMigrator(self._soledad)
+        return migrator.migrate()
 
     def _wait_indexes(self, *methods):
         """
@@ -343,7 +357,7 @@ class EncryptionScheme(object):
             self.stored[method] = getattr(self, method)
             setattr(self, method, makeWrapper(method))
 
-        self.deferred_indexes.addCallback(restore)
+        self.deferred_init.addCallback(restore)
 
     @abstractmethod
     def get_key(self, address, private=False):
diff --git a/src/leap/keymanager/migrator.py b/src/leap/keymanager/migrator.py
new file mode 100644
index 0000000..b59647a
--- /dev/null
+++ b/src/leap/keymanager/migrator.py
@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+# migrator.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Document migrator
+"""
+# XXX: versioning has being added 12/2015 when keymanager was not
+#      much in use in the wild. We can probably drop support for
+#      keys without version at some point.
+
+
+from collections import namedtuple
+from twisted.internet.defer import gatherResults, succeed
+
+from leap.keymanager.keys import (
+    TAGS_PRIVATE_INDEX,
+    KEYMANAGER_KEY_TAG,
+    KEYMANAGER_ACTIVE_TAG,
+
+    KEYMANAGER_DOC_VERSION,
+    KEY_VERSION_KEY,
+    KEY_ID_KEY,
+    KEY_VALIDATION_KEY,
+    KEY_LAST_AUDITED_AT_KEY,
+    KEY_ENCR_USED_KEY,
+    KEY_SIGN_USED_KEY,
+)
+from leap.keymanager.validation import ValidationLevels
+
+
+KeyDocs = namedtuple("KeyDocs", ['key', 'active'])
+
+
+class KeyDocumentsMigrator(object):
+    """
+    Migrate old KeyManager Soledad Documents to the newest schema
+    """
+
+    def __init__(self, soledad):
+        self._soledad = soledad
+
+    def migrate(self):
+        deferred_public = self._get_docs(private=False)
+        deferred_public.addCallback(self._migrate_docs)
+
+        deferred_private = self._get_docs(private=True)
+        deferred_private.addCallback(self._migrate_docs)
+
+        return gatherResults([deferred_public, deferred_private])
+
+    def _get_docs(self, private=False):
+        private_value = '1' if private else '0'
+
+        deferred_keys = self._soledad.get_from_index(
+            TAGS_PRIVATE_INDEX,
+            KEYMANAGER_KEY_TAG,
+            private_value)
+        deferred_active = self._soledad.get_from_index(
+            TAGS_PRIVATE_INDEX,
+            KEYMANAGER_ACTIVE_TAG,
+            private_value)
+        return gatherResults([deferred_keys, deferred_active])
+
+    def _migrate_docs(self, (key_docs, active_docs)):
+        def update_keys(keys):
+            deferreds = []
+            for key_id in keys:
+                key = keys[key_id].key
+                actives = keys[key_id].active
+
+                d = self._migrate_actives(key, actives)
+                deferreds.append(d)
+
+                d = self._migrate_key(key)
+                deferreds.append(d)
+            return gatherResults(deferreds)
+
+        d = self._buildKeyDict(key_docs, active_docs)
+        d.addCallback(lambda keydict: self._filter_outdated(keydict))
+        d.addCallback(update_keys)
+
+    def _buildKeyDict(self, keys, actives):
+        keydict = {
+            fp2id(key.content[KEY_FINGERPRINT_KEY]): KeyDocs(key, [])
+            for key in keys}
+
+        deferreds = []
+        for active in actives:
+            if KEY_ID_KEY in active.content:
+                key_id = active.content[KEY_ID_KEY]
+                if key_id not in keydict:
+                    d = self._soledad.delete_doc(active)
+                    deferreds.append(d)
+                    continue
+                keydict[key_id].active.append(active)
+
+        d = gatherResults(deferreds)
+        d.addCallback(lambda _: keydict)
+        return d
+
+    def _filter_outdated(self, keydict):
+        outdated = {}
+        for key_id, docs in keydict.items():
+            if ((docs.key and KEY_VERSION_KEY not in docs.key.content) or
+                    docs.active):
+                outdated[key_id] = docs
+        return outdated
+
+    def _migrate_actives(self, key, actives):
+        if not key:
+            deferreds = []
+            for active in actives:
+                d = self._soledad.delete_doc(active)
+                deferreds.append(d)
+            return gatherResults(deferreds)
+
+        validation = str(ValidationLevels.Weak_Chain)
+        last_audited = 0
+        encr_used = False
+        sign_used = False
+        if len(actives) == 1 and KEY_VERSION_KEY not in key.content:
+            # we can preserve the validation of the key if there is only one
+            # active address for the key
+            validation = key.content[KEY_VALIDATION_KEY]
+            last_audited = key.content[KEY_LAST_AUDITED_AT_KEY]
+            encr_used = key.content[KEY_ENCR_USED_KEY]
+            sign_used = key.content[KEY_SIGN_USED_KEY]
+
+        deferreds = []
+        for active in actives:
+            if KEY_VERSION_KEY in active.content:
+                continue
+
+            active.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+            active.content[KEY_VALIDATION_KEY] = validation
+            active.content[KEY_LAST_AUDITED_AT_KEY] = last_audited
+            active.content[KEY_ENCR_USED_KEY] = encr_used
+            active.content[KEY_SIGN_USED_KEY] = sign_used
+            d = self._soledad.put_doc(active)
+            deferreds.append(d)
+        return gatherResults(deferreds)
+
+    def _migrate_key(self, key):
+        if not key or KEY_VERSION_KEY in key.content:
+            return succeed(None)
+
+        key.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+        del key.content[KEY_VALIDATION_KEY]
+        del key.content[KEY_LAST_AUDITED_AT_KEY]
+        del key.content[KEY_ENCR_USED_KEY]
+        del key.content[KEY_SIGN_USED_KEY]
+        return self._soledad.put_doc(key)
+
+
+def fp2id(fingerprint):
+    KEY_ID_LENGTH = 16
+    return fingerprint[-KEY_ID_LENGTH:]
diff --git a/src/leap/keymanager/tests/__init__.py b/src/leap/keymanager/tests/__init__.py
index cd612c4..d02f187 100644
--- a/src/leap/keymanager/tests/__init__.py
+++ b/src/leap/keymanager/tests/__init__.py
@@ -76,7 +76,7 @@ class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
             return d
 
         # wait for the indexes to be ready for the tear down
-        d = km._wrapper_map[OpenPGPKey].deferred_indexes
+        d = km._wrapper_map[OpenPGPKey].deferred_init
         d.addCallback(get_and_delete_keys)
         d.addCallback(lambda _: self.tearDownEnv())
         d.addCallback(lambda _: self._soledad.close())
-- 
cgit v1.2.3


From 3d544f4a85930c5d1611d193500744fc97f0aee1 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Fri, 18 Dec 2015 20:31:18 +0100
Subject: [feat] Use fingerprints instead of key ids

- Resolves: #7500
---
 changes/next-changelog.txt                   |   3 +
 src/leap/keymanager/__init__.py              |   6 +-
 src/leap/keymanager/keys.py                  |  21 ++---
 src/leap/keymanager/migrator.py              |   8 +-
 src/leap/keymanager/openpgp.py               |  43 +++++-----
 src/leap/keymanager/tests/__init__.py        |   1 -
 src/leap/keymanager/tests/test_keymanager.py |   5 --
 src/leap/keymanager/tests/test_openpgp.py    | 123 +++++++++++----------------
 src/leap/keymanager/validation.py            |   4 +-
 9 files changed, 94 insertions(+), 120 deletions(-)

diff --git a/changes/next-changelog.txt b/changes/next-changelog.txt
index de837da..a53a5d2 100644
--- a/changes/next-changelog.txt
+++ b/changes/next-changelog.txt
@@ -12,6 +12,7 @@ Features
 ~~~~~~~~
 - `#7485 <https://leap.se/code/issues/7485>`_: Move validation, usage and audited date to the active document.
 - `#7713 <https://leap.se/code/issues/7713>`_: Update soledad documents by adding versioning field.
+- `#7500 <https://leap.se/code/issues/7500>`_: Use fingerprints instead of key ids.
 
 - `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
 - New feature without related issue number.
@@ -23,6 +24,8 @@ Bugfixes
 
 Misc
 ~~~~
+- This version includes changes in the Soledad Documents and minor modifications to the API.
+
 - `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
 - Some change without issue number.
 
diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 8b3487f..8a4efbe 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -642,7 +642,7 @@ class KeyManager(object):
             else:
                 signature = InvalidSignature(
                     'Failed to verify signature with key %s' %
-                    (pubkey.key_id,))
+                    (pubkey.fingerprint,))
             defer.returnValue((decrypted, signature))
 
         dpriv = self.get_key(address, ktype, private=True)
@@ -741,7 +741,7 @@ class KeyManager(object):
             else:
                 raise InvalidSignature(
                     'Failed to verify signature with key %s' %
-                    (pubkey.key_id,))
+                    (pubkey.fingerprint,))
 
         d = self.get_key(address, ktype, private=False,
                          fetch_remote=fetch_remote)
@@ -804,7 +804,7 @@ class KeyManager(object):
             else:
                 raise KeyNotValidUpgrade(
                     "Key %s can not be upgraded by new key %s"
-                    % (old_key.key_id, key.key_id))
+                    % (old_key.fingerprint, key.fingerprint))
 
         d = _keys.get_key(address, private=key.private)
         d.addErrback(old_key_not_found)
diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index a60c19d..68e3fad 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -48,7 +48,6 @@ logger = logging.getLogger(__name__)
 KEY_VERSION_KEY = 'version'
 KEY_ADDRESS_KEY = 'address'
 KEY_TYPE_KEY = 'type'
-KEY_ID_KEY = 'key_id'
 KEY_FINGERPRINT_KEY = 'fingerprint'
 KEY_DATA_KEY = 'key_data'
 KEY_PRIVATE_KEY = 'private'
@@ -80,16 +79,16 @@ KEYMANAGER_DOC_VERSION = 1
 #
 
 TAGS_PRIVATE_INDEX = 'by-tags-private'
-TYPE_ID_PRIVATE_INDEX = 'by-type-id-private'
+TYPE_FINGERPRINT_PRIVATE_INDEX = 'by-type-fingerprint-private'
 TYPE_ADDRESS_PRIVATE_INDEX = 'by-type-address-private'
 INDEXES = {
     TAGS_PRIVATE_INDEX: [
         KEY_TAGS_KEY,
         'bool(%s)' % KEY_PRIVATE_KEY,
     ],
-    TYPE_ID_PRIVATE_INDEX: [
+    TYPE_FINGERPRINT_PRIVATE_INDEX: [
         KEY_TYPE_KEY,
-        KEY_ID_KEY,
+        KEY_FINGERPRINT_KEY,
         'bool(%s)' % KEY_PRIVATE_KEY,
     ],
     TYPE_ADDRESS_PRIVATE_INDEX: [
@@ -137,7 +136,8 @@ def build_key_from_dict(kClass, key, active=None):
             validation = ValidationLevels.get(active[KEY_VALIDATION_KEY])
         except ValueError:
             logger.error("Not valid validation level (%s) for key %s",
-                         (active[KEY_VALIDATION_KEY], active[KEY_ID_KEY]))
+                         (active[KEY_VALIDATION_KEY],
+                          active[KEY_FINGERPRINT_KEY]))
         last_audited_at = _to_datetime(active[KEY_LAST_AUDITED_AT_KEY])
         encr_used = active[KEY_ENCR_USED_KEY]
         sign_used = active[KEY_SIGN_USED_KEY]
@@ -147,7 +147,6 @@ def build_key_from_dict(kClass, key, active=None):
 
     return kClass(
         key[KEY_ADDRESS_KEY],
-        key_id=key[KEY_ID_KEY],
         fingerprint=key[KEY_FINGERPRINT_KEY],
         key_data=key[KEY_DATA_KEY],
         private=key[KEY_PRIVATE_KEY],
@@ -189,13 +188,12 @@ class EncryptionKey(object):
 
     __metaclass__ = ABCMeta
 
-    def __init__(self, address, key_id="", fingerprint="",
+    def __init__(self, address, fingerprint="",
                  key_data="", private=False, length=0, expiry_date=None,
                  validation=ValidationLevels.Weak_Chain, last_audited_at=None,
                  refreshed_at=None, encr_used=False, sign_used=False):
         # TODO: it should know its own active address
         self.address = address
-        self.key_id = key_id
         self.fingerprint = fingerprint
         self.key_data = key_data
         self.private = private
@@ -221,7 +219,6 @@ class EncryptionKey(object):
         return json.dumps({
             KEY_ADDRESS_KEY: self.address,
             KEY_TYPE_KEY: self.__class__.__name__,
-            KEY_ID_KEY: self.key_id,
             KEY_FINGERPRINT_KEY: self.fingerprint,
             KEY_DATA_KEY: self.key_data,
             KEY_PRIVATE_KEY: self.private,
@@ -244,7 +241,7 @@ class EncryptionKey(object):
         return json.dumps({
             KEY_ADDRESS_KEY: address,
             KEY_TYPE_KEY: self.__class__.__name__ + KEYMANAGER_ACTIVE_TYPE,
-            KEY_ID_KEY: self.key_id,
+            KEY_FINGERPRINT_KEY: self.fingerprint,
             KEY_PRIVATE_KEY: self.private,
             KEY_VALIDATION_KEY: str(self.validation),
             KEY_LAST_AUDITED_AT_KEY: last_audited_at,
@@ -260,7 +257,7 @@ class EncryptionKey(object):
         """
         return u"<%s 0x%s (%s - %s)>" % (
             self.__class__.__name__,
-            self.key_id,
+            self.fingerprint,
             self.address,
             "priv" if self.private else "publ")
 
@@ -519,7 +516,7 @@ class EncryptionScheme(object):
         """
         def log_active_doc(doc):
             logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
-                                       doc.content[KEY_ID_KEY]))
+                                       doc.content[KEY_FINGERPRINT_KEY]))
 
         def cmp_active(d1, d2):
             res = cmp(d1.content[KEY_LAST_AUDITED_AT_KEY],
diff --git a/src/leap/keymanager/migrator.py b/src/leap/keymanager/migrator.py
index b59647a..11cf243 100644
--- a/src/leap/keymanager/migrator.py
+++ b/src/leap/keymanager/migrator.py
@@ -33,7 +33,7 @@ from leap.keymanager.keys import (
 
     KEYMANAGER_DOC_VERSION,
     KEY_VERSION_KEY,
-    KEY_ID_KEY,
+    KEY_FINGERPRINT_KEY,
     KEY_VALIDATION_KEY,
     KEY_LAST_AUDITED_AT_KEY,
     KEY_ENCR_USED_KEY,
@@ -42,6 +42,8 @@ from leap.keymanager.keys import (
 from leap.keymanager.validation import ValidationLevels
 
 
+KEY_ID_KEY = 'key_id'
+
 KeyDocs = namedtuple("KeyDocs", ['key', 'active'])
 
 
@@ -132,6 +134,7 @@ class KeyDocumentsMigrator(object):
         last_audited = 0
         encr_used = False
         sign_used = False
+        fingerprint = key.content[KEY_FINGERPRINT_KEY]
         if len(actives) == 1 and KEY_VERSION_KEY not in key.content:
             # we can preserve the validation of the key if there is only one
             # active address for the key
@@ -146,10 +149,12 @@ class KeyDocumentsMigrator(object):
                 continue
 
             active.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+            active.content[KEY_FINGERPRINT_KEY] = fingerprint
             active.content[KEY_VALIDATION_KEY] = validation
             active.content[KEY_LAST_AUDITED_AT_KEY] = last_audited
             active.content[KEY_ENCR_USED_KEY] = encr_used
             active.content[KEY_SIGN_USED_KEY] = sign_used
+            del active.content[KEY_ID_KEY]
             d = self._soledad.put_doc(active)
             deferreds.append(d)
         return gatherResults(deferreds)
@@ -159,6 +164,7 @@ class KeyDocumentsMigrator(object):
             return succeed(None)
 
         key.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+        del key.content[KEY_ID_KEY]
         del key.content[KEY_VALIDATION_KEY]
         del key.content[KEY_LAST_AUDITED_AT_KEY]
         del key.content[KEY_ENCR_USED_KEY]
diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index 3c8ac1e..0f16296 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -39,10 +39,10 @@ from leap.keymanager.keys import (
     EncryptionScheme,
     is_address,
     build_key_from_dict,
-    TYPE_ID_PRIVATE_INDEX,
+    TYPE_FINGERPRINT_PRIVATE_INDEX,
     TYPE_ADDRESS_PRIVATE_INDEX,
     KEY_ADDRESS_KEY,
-    KEY_ID_KEY,
+    KEY_FINGERPRINT_KEY,
     KEYMANAGER_ACTIVE_TYPE,
 )
 
@@ -122,9 +122,9 @@ class TempGPGWrapper(object):
         # itself is enough to also have the public key in the keyring,
         # and we want to count the keys afterwards.
 
-        privids = map(lambda privkey: privkey.key_id, privkeys)
+        privfps = map(lambda privkey: privkey.fingerprint, privkeys)
         publkeys = filter(
-            lambda pubkey: pubkey.key_id not in privids, publkeys)
+            lambda pubkey: pubkey.fingerprint not in privfps, publkeys)
 
         listkeys = lambda: self._gpg.list_keys()
         listsecretkeys = lambda: self._gpg.list_keys(secret=True)
@@ -213,7 +213,7 @@ class OpenPGPKey(EncryptionKey):
         :rtype: list(str)
         """
         with TempGPGWrapper(keys=[self], gpgbinary=self._gpgbinary) as gpg:
-            res = gpg.list_sigs(self.key_id)
+            res = gpg.list_sigs(self.fingerprint)
             for uid, sigs in res.sigs.iteritems():
                 if _parse_address(uid) in self.address:
                     return sigs
@@ -370,7 +370,7 @@ class OpenPGPScheme(EncryptionScheme):
             leap_assert(
                 address in keydoc.content[KEY_ADDRESS_KEY],
                 'Wrong address in key %s. Expected %s, found %s.'
-                % (keydoc.content[KEY_ID_KEY], address,
+                % (keydoc.content[KEY_FINGERPRINT_KEY], address,
                    keydoc.content[KEY_ADDRESS_KEY]))
             key = build_key_from_dict(OpenPGPKey, keydoc.content,
                                       activedoc.content)
@@ -493,7 +493,7 @@ class OpenPGPScheme(EncryptionScheme):
                 deferreds.append(d)
             return defer.gatherResults(deferreds)
 
-        dk = self._get_key_doc_from_keyid(key.key_id, key.private)
+        dk = self._get_key_doc_from_fingerprint(key.fingerprint, key.private)
         da = self._get_active_doc_from_address(address, key.private)
         d = defer.gatherResults([dk, da])
         d.addCallback(merge_and_put)
@@ -517,8 +517,8 @@ class OpenPGPScheme(EncryptionScheme):
         def get_key_from_active_doc(activedoc):
             if not activedoc:
                 return (None, None)
-            key_id = activedoc.content[KEY_ID_KEY]
-            d = self._get_key_doc_from_keyid(key_id, private)
+            fingerprint = activedoc.content[KEY_FINGERPRINT_KEY]
+            d = self._get_key_doc_from_fingerprint(fingerprint, private)
             d.addCallback(delete_active_if_no_key, activedoc)
             return d
 
@@ -573,17 +573,17 @@ class OpenPGPScheme(EncryptionScheme):
 
         def get_key_docs(_):
             return self._soledad.get_from_index(
-                TYPE_ID_PRIVATE_INDEX,
+                TYPE_FINGERPRINT_PRIVATE_INDEX,
                 self.KEY_TYPE,
-                key.key_id,
+                key.fingerprint,
                 '1' if key.private else '0')
 
         def delete_key(docs):
             if len(docs) == 0:
                 raise errors.KeyNotFound(key)
             elif len(docs) > 1:
-                logger.warning("There is more than one key for key_id %s"
-                               % key.key_id)
+                logger.warning("There is more than one key for fingerprint %s"
+                               % key.fingerprint)
 
             has_deleted = False
             deferreds = []
@@ -597,9 +597,9 @@ class OpenPGPScheme(EncryptionScheme):
             return defer.gatherResults(deferreds)
 
         d = self._soledad.get_from_index(
-            TYPE_ID_PRIVATE_INDEX,
+            TYPE_FINGERPRINT_PRIVATE_INDEX,
             self.ACTIVE_TYPE,
-            key.key_id,
+            key.fingerprint,
             '1' if key.private else '0')
         d.addCallback(delete_docs)
         d.addCallback(get_key_docs)
@@ -659,7 +659,7 @@ class OpenPGPScheme(EncryptionScheme):
             result = yield from_thread(
                 gpg.encrypt,
                 data, pubkey.fingerprint,
-                default_key=sign.key_id if sign else None,
+                default_key=sign.fingerprint if sign else None,
                 passphrase=passphrase, symmetric=False,
                 cipher_algo=cipher_algo)
             # Here we cannot assert for correctness of sig because the sig is
@@ -761,7 +761,7 @@ class OpenPGPScheme(EncryptionScheme):
         # result.fingerprint - contains the fingerprint of the key used to
         #                      sign.
         with TempGPGWrapper(privkey, self._gpgbinary) as gpg:
-            result = gpg.sign(data, default_key=privkey.key_id,
+            result = gpg.sign(data, default_key=privkey.fingerprint,
                               digest_algo=digest_algo, clearsign=clearsign,
                               detach=detach, binary=binary)
             rfprint = privkey.fingerprint
@@ -770,7 +770,7 @@ class OpenPGPScheme(EncryptionScheme):
             if result.fingerprint is None:
                 raise errors.SignFailed(
                     'Failed to sign with key %s: %s' %
-                    (privkey['keyid'], result.stderr))
+                    (privkey['fingerprint'], result.stderr))
             leap_assert(
                 result.fingerprint == kfprint,
                 'Signature and private key fingerprints mismatch: '
@@ -823,11 +823,11 @@ class OpenPGPScheme(EncryptionScheme):
         d.addCallback(self._repair_and_get_doc, self._repair_active_docs)
         return d
 
-    def _get_key_doc_from_keyid(self, key_id, private):
+    def _get_key_doc_from_fingerprint(self, fingerprint, private):
         d = self._soledad.get_from_index(
-            TYPE_ID_PRIVATE_INDEX,
+            TYPE_FINGERPRINT_PRIVATE_INDEX,
             self.KEY_TYPE,
-            key_id,
+            fingerprint,
             '1' if private else '0')
         d.addCallback(self._repair_and_get_doc, self._repair_key_docs)
         return d
@@ -863,7 +863,6 @@ def build_gpg_key(key_info, key_data, gpgbinary=None):
     return OpenPGPKey(
         address,
         gpgbinary=gpgbinary,
-        key_id=key_info['keyid'],
         fingerprint=key_info['fingerprint'],
         key_data=key_data,
         private=True if key_info['type'] == 'sec' else False,
diff --git a/src/leap/keymanager/tests/__init__.py b/src/leap/keymanager/tests/__init__.py
index d02f187..4fbf63e 100644
--- a/src/leap/keymanager/tests/__init__.py
+++ b/src/leap/keymanager/tests/__init__.py
@@ -97,7 +97,6 @@ class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
 
 
 # key 24D18DDF: public key "Leap Test Key <leap@leap.se>"
-KEY_ID = "2F455E2824D18DDF"
 KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF"
 PUBLIC_KEY = """
 -----BEGIN PGP PUBLIC KEY BLOCK-----
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index e4e0d8b..2fe9e4c 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -77,7 +77,6 @@ class KeyManagerUtilTestCase(unittest.TestCase):
     def test_build_key_from_dict(self):
         kdict = {
             'address': [ADDRESS],
-            'key_id': KEY_FINGERPRINT[-16:],
             'fingerprint': KEY_FINGERPRINT,
             'key_data': PUBLIC_KEY,
             'private': False,
@@ -87,7 +86,6 @@ class KeyManagerUtilTestCase(unittest.TestCase):
         }
         adict = {
             'address': ADDRESS,
-            'key_id': KEY_FINGERPRINT[-16:],
             'private': False,
             'last_audited_at': 0,
             'validation': str(ValidationLevels.Weak_Chain),
@@ -98,9 +96,6 @@ class KeyManagerUtilTestCase(unittest.TestCase):
         self.assertEqual(
             kdict['address'], key.address,
             'Wrong data in key.')
-        self.assertEqual(
-            kdict['key_id'], key.key_id,
-            'Wrong data in key.')
         self.assertEqual(
             kdict['fingerprint'], key.fingerprint,
             'Wrong data in key.')
diff --git a/src/leap/keymanager/tests/test_openpgp.py b/src/leap/keymanager/tests/test_openpgp.py
index 6641591..8ed049f 100644
--- a/src/leap/keymanager/tests/test_openpgp.py
+++ b/src/leap/keymanager/tests/test_openpgp.py
@@ -30,7 +30,7 @@ from leap.keymanager import (
     openpgp,
 )
 from leap.keymanager.keys import (
-    TYPE_ID_PRIVATE_INDEX,
+    TYPE_FINGERPRINT_PRIVATE_INDEX,
     TYPE_ADDRESS_PRIVATE_INDEX,
 )
 from leap.keymanager.openpgp import OpenPGPKey
@@ -40,7 +40,6 @@ from leap.keymanager.tests import (
     ADDRESS_2,
     KEY_FINGERPRINT,
     PUBLIC_KEY,
-    KEY_ID,
     PUBLIC_KEY_2,
     PRIVATE_KEY,
     PRIVATE_KEY_2,
@@ -256,39 +255,18 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
 
     @inlineCallbacks
     def test_self_repair_three_keys(self):
+        refreshed_keep = datetime(2007, 1, 1)
+        self._insert_key_docs([datetime(2005, 1, 1),
+                               refreshed_keep,
+                               datetime(2001, 1, 1)])
+        delete_doc = self._mock_delete_doc()
+
         pgp = openpgp.OpenPGPScheme(
             self._soledad, gpgbinary=self.gpg_binary_path)
-        yield pgp.put_ascii_key(PUBLIC_KEY, ADDRESS)
-
-        get_from_index = self._soledad.get_from_index
-        delete_doc = self._soledad.delete_doc
-
-        def my_get_from_index(*args):
-            if (args[0] == TYPE_ID_PRIVATE_INDEX and
-                    args[2] == KEY_ID):
-                k1 = OpenPGPKey(ADDRESS, key_id="1",
-                                refreshed_at=datetime(2005, 1, 1))
-                k2 = OpenPGPKey(ADDRESS, key_id="2",
-                                refreshed_at=datetime(2007, 1, 1))
-                k3 = OpenPGPKey(ADDRESS, key_id="3",
-                                refreshed_at=datetime(2001, 1, 1))
-                d1 = self._soledad.create_doc_from_json(k1.get_json())
-                d2 = self._soledad.create_doc_from_json(k2.get_json())
-                d3 = self._soledad.create_doc_from_json(k3.get_json())
-                return gatherResults([d1, d2, d3])
-            return get_from_index(*args)
-
-        self._soledad.get_from_index = my_get_from_index
-        self._soledad.delete_doc = Mock(return_value=succeed(None))
-
         key = yield pgp.get_key(ADDRESS, private=False)
-
-        try:
-            self.assertEqual(key.key_id, "2")
-            self.assertEqual(self._soledad.delete_doc.call_count, 2)
-        finally:
-            self._soledad.get_from_index = get_from_index
-            self._soledad.delete_doc = delete_doc
+        self.assertEqual(key.refreshed_at, refreshed_keep)
+        self.assertEqual(self.count, 2)
+        self._soledad.delete_doc = delete_doc
 
     @inlineCallbacks
     def test_self_repair_no_keys(self):
@@ -300,8 +278,8 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         delete_doc = self._soledad.delete_doc
 
         def my_get_from_index(*args):
-            if (args[0] == TYPE_ID_PRIVATE_INDEX and
-                    args[2] == KEY_ID):
+            if (args[0] == TYPE_FINGERPRINT_PRIVATE_INDEX and
+                    args[2] == KEY_FINGERPRINT):
                 return succeed([])
             return get_from_index(*args)
 
@@ -319,39 +297,16 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
 
     @inlineCallbacks
     def test_self_repair_put_keys(self):
+        self._insert_key_docs([datetime(2005, 1, 1),
+                               datetime(2007, 1, 1),
+                               datetime(2001, 1, 1)])
+        delete_doc = self._mock_delete_doc()
+
         pgp = openpgp.OpenPGPScheme(
             self._soledad, gpgbinary=self.gpg_binary_path)
-
-        get_from_index = self._soledad.get_from_index
-        delete_doc = self._soledad.delete_doc
-
-        def my_get_from_index(*args):
-            if (args[0] == TYPE_ID_PRIVATE_INDEX and
-                    args[2] == KEY_ID):
-                k1 = OpenPGPKey(ADDRESS, key_id="1",
-                                fingerprint=KEY_FINGERPRINT,
-                                refreshed_at=datetime(2005, 1, 1))
-                k2 = OpenPGPKey(ADDRESS, key_id="2",
-                                fingerprint=KEY_FINGERPRINT,
-                                refreshed_at=datetime(2007, 1, 1))
-                k3 = OpenPGPKey(ADDRESS, key_id="3",
-                                fingerprint=KEY_FINGERPRINT,
-                                refreshed_at=datetime(2001, 1, 1))
-                d1 = self._soledad.create_doc_from_json(k1.get_json())
-                d2 = self._soledad.create_doc_from_json(k2.get_json())
-                d3 = self._soledad.create_doc_from_json(k3.get_json())
-                return gatherResults([d1, d2, d3])
-            return get_from_index(*args)
-
-        self._soledad.get_from_index = my_get_from_index
-        self._soledad.delete_doc = Mock(return_value=succeed(None))
-
-        try:
-            yield pgp.put_ascii_key(PUBLIC_KEY, ADDRESS)
-            self.assertEqual(self._soledad.delete_doc.call_count, 2)
-        finally:
-            self._soledad.get_from_index = get_from_index
-            self._soledad.delete_doc = delete_doc
+        yield pgp.put_ascii_key(PUBLIC_KEY, ADDRESS)
+        self.assertEqual(self.count, 2)
+        self._soledad.delete_doc = delete_doc
 
     @inlineCallbacks
     def test_self_repair_five_active_docs(self):
@@ -364,29 +319,29 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         def my_get_from_index(*args):
             if (args[0] == TYPE_ADDRESS_PRIVATE_INDEX and
                     args[2] == ADDRESS):
-                k1 = OpenPGPKey(ADDRESS, key_id="1",
+                k1 = OpenPGPKey(ADDRESS, fingerprint="1",
                                 last_audited_at=datetime(2005, 1, 1))
-                k2 = OpenPGPKey(ADDRESS, key_id="2",
+                k2 = OpenPGPKey(ADDRESS, fingerprint="2",
                                 last_audited_at=datetime(2007, 1, 1))
-                k3 = OpenPGPKey(ADDRESS, key_id="3",
+                k3 = OpenPGPKey(ADDRESS, fingerprint="3",
                                 last_audited_at=datetime(2007, 1, 1),
                                 encr_used=True, sign_used=True)
-                k4 = OpenPGPKey(ADDRESS, key_id="4",
+                k4 = OpenPGPKey(ADDRESS, fingerprint="4",
                                 last_audited_at=datetime(2007, 1, 1),
                                 sign_used=True)
-                k5 = OpenPGPKey(ADDRESS, key_id="5",
+                k5 = OpenPGPKey(ADDRESS, fingerprint="5",
                                 last_audited_at=datetime(2007, 1, 1),
                                 encr_used=True)
                 deferreds = []
-                for k in [k1, k2, k3, k4, k5]:
+                for k in (k1, k2, k3, k4, k5):
                     d = self._soledad.create_doc_from_json(
                         k.get_active_json(ADDRESS))
                     deferreds.append(d)
                 return gatherResults(deferreds)
-            elif args[0] == TYPE_ID_PRIVATE_INDEX:
-                key_id = args[2]
-                self.assertEqual(key_id, "3")
-                k = OpenPGPKey(ADDRESS, key_id="3")
+            elif args[0] == TYPE_FINGERPRINT_PRIVATE_INDEX:
+                fingerprint = args[2]
+                self.assertEqual(fingerprint, "3")
+                k = OpenPGPKey(ADDRESS, fingerprint="3")
                 return succeed(
                     [self._soledad.create_doc_from_json(k.get_json())])
             return get_from_index(*args)
@@ -404,3 +359,21 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
     def _assert_key_not_found(self, pgp, address, private=False):
         d = pgp.get_key(address, private=private)
         return self.assertFailure(d, KeyNotFound)
+
+    @inlineCallbacks
+    def _insert_key_docs(self, refreshed_at):
+        for date in refreshed_at:
+            key = OpenPGPKey(ADDRESS, fingerprint=KEY_FINGERPRINT,
+                             refreshed_at=date)
+            yield self._soledad.create_doc_from_json(key.get_json())
+        yield self._soledad.create_doc_from_json(key.get_active_json())
+
+    def _mock_delete_doc(self):
+        delete_doc = self._soledad.delete_doc
+        self.count = 0
+
+        def my_delete_doc(*args):
+            self.count += 1
+            return delete_doc(*args)
+        self._soledad.delete_doc = my_delete_doc
+        return delete_doc
diff --git a/src/leap/keymanager/validation.py b/src/leap/keymanager/validation.py
index 734cfce..8cf96da 100644
--- a/src/leap/keymanager/validation.py
+++ b/src/leap/keymanager/validation.py
@@ -118,7 +118,9 @@ def can_upgrade(new_key, old_key):
         return True
 
     # New key signed by the old key
-    if old_key.key_id in new_key.signatures:
+    # XXX: signatures are using key-ids instead of fingerprints
+    key_id = old_key.fingerprint[-16:]
+    if key_id in new_key.signatures:
         return True
 
     return False
-- 
cgit v1.2.3


From 6222996d0805dfec1fab949b536adb0af08df0be Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Mon, 21 Dec 2015 20:24:52 +0100
Subject: [test] add updater tests

---
 src/leap/keymanager/tests/test_migrator.py | 175 +++++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)
 create mode 100644 src/leap/keymanager/tests/test_migrator.py

diff --git a/src/leap/keymanager/tests/test_migrator.py b/src/leap/keymanager/tests/test_migrator.py
new file mode 100644
index 0000000..2d9782b
--- /dev/null
+++ b/src/leap/keymanager/tests/test_migrator.py
@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+# test_migrator.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Tests for the migrator.
+"""
+
+
+from collections import namedtuple
+from mock import Mock
+from twisted.internet.defer import succeed, inlineCallbacks
+
+from leap.keymanager.migrator import KeyDocumentsMigrator, KEY_ID_KEY
+from leap.keymanager.keys import (
+    TAGS_PRIVATE_INDEX,
+    KEYMANAGER_ACTIVE_TAG,
+    KEYMANAGER_KEY_TAG,
+    KEYMANAGER_DOC_VERSION,
+
+    KEY_ADDRESS_KEY,
+    KEY_UIDS_KEY,
+    KEY_VERSION_KEY,
+    KEY_FINGERPRINT_KEY,
+    KEY_VALIDATION_KEY,
+    KEY_LAST_AUDITED_AT_KEY,
+    KEY_ENCR_USED_KEY,
+    KEY_SIGN_USED_KEY,
+)
+from leap.keymanager.validation import ValidationLevels
+from leap.keymanager.tests import (
+    KeyManagerWithSoledadTestCase,
+    ADDRESS,
+    ADDRESS_2,
+    KEY_FINGERPRINT,
+)
+
+
+class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
+    @inlineCallbacks
+    def test_simple_migration(self):
+        get_from_index = self._soledad.get_from_index
+        delete_doc = self._soledad.delete_doc
+        put_doc = self._soledad.put_doc
+
+        def my_get_from_index(*args):
+            docs = []
+            if (args[0] == TAGS_PRIVATE_INDEX and
+                    args[2] == '0'):
+                SoledadDocument = namedtuple("SoledadDocument", ["content"])
+                if args[1] == KEYMANAGER_KEY_TAG:
+                    docs = [SoledadDocument({
+                        KEY_ADDRESS_KEY: [ADDRESS],
+                        KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                        KEY_FINGERPRINT_KEY: KEY_FINGERPRINT,
+                        KEY_VALIDATION_KEY: str(ValidationLevels.Weak_Chain),
+                        KEY_LAST_AUDITED_AT_KEY: 0,
+                        KEY_ENCR_USED_KEY: True,
+                        KEY_SIGN_USED_KEY: False,
+                    })]
+                if args[1] == KEYMANAGER_ACTIVE_TAG:
+                    docs = [SoledadDocument({
+                        KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                    })]
+            return succeed(docs)
+
+        self._soledad.get_from_index = my_get_from_index
+        self._soledad.delete_doc = Mock(return_value=succeed(None))
+        self._soledad.put_doc = Mock(return_value=succeed(None))
+
+        try:
+            migrator = KeyDocumentsMigrator(self._soledad)
+            yield migrator.migrate()
+            call_list = self._soledad.put_doc.call_args_list
+        finally:
+            self._soledad.get_from_index = get_from_index
+            self._soledad.delete_doc = delete_doc
+            self._soledad.put_doc = put_doc
+
+        self.assertEqual(len(call_list), 2)
+        active = call_list[0][0][0]
+        key = call_list[1][0][0]
+
+        self.assertTrue(KEY_ID_KEY not in active.content)
+        self.assertEqual(active.content[KEY_VERSION_KEY],
+                         KEYMANAGER_DOC_VERSION)
+        self.assertEqual(active.content[KEY_FINGERPRINT_KEY], KEY_FINGERPRINT)
+        self.assertEqual(active.content[KEY_VALIDATION_KEY],
+                         str(ValidationLevels.Weak_Chain))
+        self.assertEqual(active.content[KEY_LAST_AUDITED_AT_KEY], 0)
+        self.assertEqual(active.content[KEY_ENCR_USED_KEY], True)
+        self.assertEqual(active.content[KEY_SIGN_USED_KEY], False)
+
+        self.assertTrue(KEY_ID_KEY not in key.content)
+        self.assertTrue(KEY_ADDRESS_KEY not in key.content)
+        self.assertTrue(KEY_VALIDATION_KEY not in key.content)
+        self.assertTrue(KEY_LAST_AUDITED_AT_KEY not in key.content)
+        self.assertTrue(KEY_ENCR_USED_KEY not in key.content)
+        self.assertTrue(KEY_SIGN_USED_KEY not in key.content)
+        self.assertEqual(key.content[KEY_UIDS_KEY], [ADDRESS])
+
+    @inlineCallbacks
+    def test_two_active_docs(self):
+        get_from_index = self._soledad.get_from_index
+        delete_doc = self._soledad.delete_doc
+        put_doc = self._soledad.put_doc
+
+        def my_get_from_index(*args):
+            docs = []
+            if (args[0] == TAGS_PRIVATE_INDEX and
+                    args[2] == '0'):
+                SoledadDocument = namedtuple("SoledadDocument", ["content"])
+                if args[1] == KEYMANAGER_KEY_TAG:
+                    validation = str(ValidationLevels.Provider_Trust)
+                    docs = [SoledadDocument({
+                        KEY_ADDRESS_KEY: [ADDRESS, ADDRESS_2],
+                        KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                        KEY_FINGERPRINT_KEY: KEY_FINGERPRINT,
+                        KEY_VALIDATION_KEY: validation,
+                        KEY_LAST_AUDITED_AT_KEY: 1984,
+                        KEY_ENCR_USED_KEY: True,
+                        KEY_SIGN_USED_KEY: False,
+                    })]
+                if args[1] == KEYMANAGER_ACTIVE_TAG:
+                    docs = [
+                        SoledadDocument({
+                            KEY_ADDRESS_KEY: ADDRESS,
+                            KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                        }),
+                        SoledadDocument({
+                            KEY_ADDRESS_KEY: ADDRESS_2,
+                            KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                        }),
+                    ]
+            return succeed(docs)
+
+        self._soledad.get_from_index = my_get_from_index
+        self._soledad.delete_doc = Mock(return_value=succeed(None))
+        self._soledad.put_doc = Mock(return_value=succeed(None))
+
+        try:
+            migrator = KeyDocumentsMigrator(self._soledad)
+            yield migrator.migrate()
+            call_list = self._soledad.put_doc.call_args_list
+        finally:
+            self._soledad.get_from_index = get_from_index
+            self._soledad.delete_doc = delete_doc
+            self._soledad.put_doc = put_doc
+
+        self.assertEqual(len(call_list), 3)
+        for active in [call[0][0] for call in call_list][:2]:
+            self.assertTrue(KEY_ID_KEY not in active.content)
+            self.assertEqual(active.content[KEY_VERSION_KEY],
+                             KEYMANAGER_DOC_VERSION)
+            self.assertEqual(active.content[KEY_FINGERPRINT_KEY],
+                             KEY_FINGERPRINT)
+            self.assertEqual(active.content[KEY_VALIDATION_KEY],
+                             str(ValidationLevels.Weak_Chain))
+            self.assertEqual(active.content[KEY_LAST_AUDITED_AT_KEY], 0)
+            self.assertEqual(active.content[KEY_ENCR_USED_KEY], False)
+            self.assertEqual(active.content[KEY_SIGN_USED_KEY], False)
-- 
cgit v1.2.3


From fdb6e285a97d5af21c7b3bdc02cba6fc21382f74 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Mon, 21 Dec 2015 19:26:55 +0100
Subject: [feat] Make EncryptionKey aware of the active address

---
 changes/next-changelog.txt                   |  1 +
 src/leap/keymanager/__init__.py              | 35 +++++------
 src/leap/keymanager/keys.py                  | 25 ++++----
 src/leap/keymanager/migrator.py              |  4 ++
 src/leap/keymanager/openpgp.py               | 86 ++++++++++++++++------------
 src/leap/keymanager/tests/test_keymanager.py | 21 ++++---
 src/leap/keymanager/tests/test_openpgp.py    |  2 +-
 7 files changed, 96 insertions(+), 78 deletions(-)

diff --git a/changes/next-changelog.txt b/changes/next-changelog.txt
index a53a5d2..be6da72 100644
--- a/changes/next-changelog.txt
+++ b/changes/next-changelog.txt
@@ -13,6 +13,7 @@ Features
 - `#7485 <https://leap.se/code/issues/7485>`_: Move validation, usage and audited date to the active document.
 - `#7713 <https://leap.se/code/issues/7713>`_: Update soledad documents by adding versioning field.
 - `#7500 <https://leap.se/code/issues/7500>`_: Use fingerprints instead of key ids.
+- Make EncryptionKey aware of the active address.
 
 - `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
 - New feature without related issue number.
diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 8a4efbe..99ee163 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -580,7 +580,7 @@ class KeyManager(object):
                 data, pubkey, passphrase, sign=signkey,
                 cipher_algo=cipher_algo)
             pubkey.encr_used = True
-            yield _keys.put_key(pubkey, address)
+            yield _keys.put_key(pubkey)
             defer.returnValue(encrypted)
 
         dpub = self.get_key(address, ktype, private=False,
@@ -637,7 +637,7 @@ class KeyManager(object):
                 signature = pubkey
                 if not pubkey.sign_used:
                     pubkey.sign_used = True
-                    yield _keys.put_key(pubkey, verify)
+                    yield _keys.put_key(pubkey)
                     defer.returnValue((decrypted, signature))
             else:
                 signature = InvalidSignature(
@@ -734,7 +734,7 @@ class KeyManager(object):
             if signed:
                 if not pubkey.sign_used:
                     pubkey.sign_used = True
-                    d = _keys.put_key(pubkey, address)
+                    d = _keys.put_key(pubkey)
                     d.addCallback(lambda _: pubkey)
                     return d
                 return pubkey
@@ -765,20 +765,16 @@ class KeyManager(object):
         _keys = self._wrapper_map[type(key)]
         return _keys.delete_key(key)
 
-    def put_key(self, key, address):
+    def put_key(self, key):
         """
         Put key bound to address in local storage.
 
         :param key: The key to be stored
         :type key: EncryptionKey
-        :param address: address for which this key will be active
-        :type address: str
 
         :return: A Deferred which fires when the key is in the storage, or
-                 which fails with KeyAddressMismatch if address doesn't match
-                 any uid on the key or fails with KeyNotValidUpdate if a key
-                 with the same uid exists and the new one is not a valid update
-                 for it.
+                 which fails with KeyNotValidUpdate if a key with the same
+                 uid exists and the new one is not a valid update for it.
         :rtype: Deferred
 
         :raise UnsupportedKeyTypeError: if invalid key type
@@ -787,11 +783,6 @@ class KeyManager(object):
         self._assert_supported_key_type(ktype)
         _keys = self._wrapper_map[ktype]
 
-        if address not in key.address:
-            return defer.fail(
-                KeyAddressMismatch("UID %s found, but expected %s"
-                                   % (str(key.address), address)))
-
         def old_key_not_found(failure):
             if failure.check(KeyNotFound):
                 return None
@@ -800,13 +791,13 @@ class KeyManager(object):
 
         def check_upgrade(old_key):
             if key.private or can_upgrade(key, old_key):
-                return _keys.put_key(key, address)
+                return _keys.put_key(key)
             else:
                 raise KeyNotValidUpgrade(
                     "Key %s can not be upgraded by new key %s"
                     % (old_key.fingerprint, key.fingerprint))
 
-        d = _keys.get_key(address, private=key.private)
+        d = _keys.get_key(key.address, private=key.private)
         d.addErrback(old_key_not_found)
         d.addCallback(check_upgrade)
         return d
@@ -838,11 +829,11 @@ class KeyManager(object):
         self._assert_supported_key_type(ktype)
         _keys = self._wrapper_map[ktype]
 
-        pubkey, privkey = _keys.parse_ascii_key(key)
+        pubkey, privkey = _keys.parse_ascii_key(key, address)
         pubkey.validation = validation
-        d = self.put_key(pubkey, address)
+        d = self.put_key(pubkey)
         if privkey is not None:
-            d.addCallback(lambda _: self.put_key(privkey, address))
+            d.addCallback(lambda _: self.put_key(privkey))
         return d
 
     @defer.inlineCallbacks
@@ -878,12 +869,12 @@ class KeyManager(object):
         ascii_content = yield self._get_with_combined_ca_bundle(uri)
 
         # XXX parse binary keys
-        pubkey, _ = _keys.parse_ascii_key(ascii_content)
+        pubkey, _ = _keys.parse_ascii_key(ascii_content, address)
         if pubkey is None:
             raise KeyNotFound(uri)
 
         pubkey.validation = validation
-        yield self.put_key(pubkey, address)
+        yield self.put_key(pubkey)
 
     def _assert_supported_key_type(self, ktype):
         """
diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index 68e3fad..38d66b5 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -46,6 +46,7 @@ logger = logging.getLogger(__name__)
 #
 
 KEY_VERSION_KEY = 'version'
+KEY_UIDS_KEY = 'uids'
 KEY_ADDRESS_KEY = 'address'
 KEY_TYPE_KEY = 'type'
 KEY_FINGERPRINT_KEY = 'fingerprint'
@@ -126,12 +127,14 @@ def build_key_from_dict(kClass, key, active=None):
     :return: An instance of the key.
     :rtype: C{kClass}
     """
+    address = None
     validation = ValidationLevels.Weak_Chain
     last_audited_at = None
     encr_used = False
     sign_used = False
 
     if active:
+        address = active[KEY_ADDRESS_KEY]
         try:
             validation = ValidationLevels.get(active[KEY_VALIDATION_KEY])
         except ValueError:
@@ -146,7 +149,8 @@ def build_key_from_dict(kClass, key, active=None):
     refreshed_at = _to_datetime(key[KEY_REFRESHED_AT_KEY])
 
     return kClass(
-        key[KEY_ADDRESS_KEY],
+        address=address,
+        uids=key[KEY_UIDS_KEY],
         fingerprint=key[KEY_FINGERPRINT_KEY],
         key_data=key[KEY_DATA_KEY],
         private=key[KEY_PRIVATE_KEY],
@@ -188,12 +192,15 @@ class EncryptionKey(object):
 
     __metaclass__ = ABCMeta
 
-    def __init__(self, address, fingerprint="",
+    def __init__(self, address=None, uids=[], fingerprint="",
                  key_data="", private=False, length=0, expiry_date=None,
                  validation=ValidationLevels.Weak_Chain, last_audited_at=None,
                  refreshed_at=None, encr_used=False, sign_used=False):
-        # TODO: it should know its own active address
         self.address = address
+        if not uids and address:
+            self.uids = [address]
+        else:
+            self.uids = uids
         self.fingerprint = fingerprint
         self.key_data = key_data
         self.private = private
@@ -217,7 +224,7 @@ class EncryptionKey(object):
         refreshed_at = _to_unix_time(self.refreshed_at)
 
         return json.dumps({
-            KEY_ADDRESS_KEY: self.address,
+            KEY_UIDS_KEY: self.uids,
             KEY_TYPE_KEY: self.__class__.__name__,
             KEY_FINGERPRINT_KEY: self.fingerprint,
             KEY_DATA_KEY: self.key_data,
@@ -229,7 +236,7 @@ class EncryptionKey(object):
             KEY_TAGS_KEY: [KEYMANAGER_KEY_TAG],
         })
 
-    def get_active_json(self, address):
+    def get_active_json(self):
         """
         Return a JSON string describing this key.
 
@@ -239,7 +246,7 @@ class EncryptionKey(object):
         last_audited_at = _to_unix_time(self.last_audited_at)
 
         return json.dumps({
-            KEY_ADDRESS_KEY: address,
+            KEY_ADDRESS_KEY: self.address,
             KEY_TYPE_KEY: self.__class__.__name__ + KEYMANAGER_ACTIVE_TYPE,
             KEY_FINGERPRINT_KEY: self.fingerprint,
             KEY_PRIVATE_KEY: self.private,
@@ -374,14 +381,12 @@ class EncryptionScheme(object):
         pass
 
     @abstractmethod
-    def put_key(self, key, address):
+    def put_key(self, key):
         """
         Put a key in local storage.
 
         :param key: The key to be stored.
         :type key: EncryptionKey
-        :param address: address for which this key will be active.
-        :type address: str
 
         :return: A Deferred which fires when the key is in the storage.
         :rtype: Deferred
@@ -496,7 +501,7 @@ class EncryptionScheme(object):
         :rtype: Deferred
         """
         def log_key_doc(doc):
-            logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
+            logger.error("\t%s: %s" % (doc.content[KEY_UIDS_KEY],
                                        doc.content[KEY_FINGERPRINT_KEY]))
 
         def cmp_key(d1, d2):
diff --git a/src/leap/keymanager/migrator.py b/src/leap/keymanager/migrator.py
index 11cf243..9e4ae77 100644
--- a/src/leap/keymanager/migrator.py
+++ b/src/leap/keymanager/migrator.py
@@ -32,6 +32,8 @@ from leap.keymanager.keys import (
     KEYMANAGER_ACTIVE_TAG,
 
     KEYMANAGER_DOC_VERSION,
+    KEY_ADDRESS_KEY,
+    KEY_UIDS_KEY,
     KEY_VERSION_KEY,
     KEY_FINGERPRINT_KEY,
     KEY_VALIDATION_KEY,
@@ -164,6 +166,8 @@ class KeyDocumentsMigrator(object):
             return succeed(None)
 
         key.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+        key.content[KEY_UIDS_KEY] = key.content[KEY_ADDRESS_KEY]
+        del key.content[KEY_ADDRESS_KEY]
         del key.content[KEY_ID_KEY]
         del key.content[KEY_VALIDATION_KEY]
         del key.content[KEY_LAST_AUDITED_AT_KEY]
diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index 0f16296..0d5a866 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -41,7 +41,7 @@ from leap.keymanager.keys import (
     build_key_from_dict,
     TYPE_FINGERPRINT_PRIVATE_INDEX,
     TYPE_ADDRESS_PRIVATE_INDEX,
-    KEY_ADDRESS_KEY,
+    KEY_UIDS_KEY,
     KEY_FINGERPRINT_KEY,
     KEYMANAGER_ACTIVE_TYPE,
 )
@@ -200,7 +200,7 @@ class OpenPGPKey(EncryptionKey):
     Base class for OpenPGP keys.
     """
 
-    def __init__(self, address, gpgbinary=None, **kwargs):
+    def __init__(self, address=None, gpgbinary=None, **kwargs):
         self._gpgbinary = gpgbinary
         super(OpenPGPKey, self).__init__(address, **kwargs)
 
@@ -215,7 +215,7 @@ class OpenPGPKey(EncryptionKey):
         with TempGPGWrapper(keys=[self], gpgbinary=self._gpgbinary) as gpg:
             res = gpg.list_sigs(self.fingerprint)
             for uid, sigs in res.sigs.iteritems():
-                if _parse_address(uid) in self.address:
+                if _parse_address(uid) in self.uids:
                     return sigs
 
         return []
@@ -335,8 +335,9 @@ class OpenPGPScheme(EncryptionScheme):
                     key = gpg.list_keys(secret=secret).pop()
                     openpgp_key = self._build_key_from_gpg(
                         key,
-                        gpg.export_keys(key['fingerprint'], secret=secret))
-                    d = self.put_key(openpgp_key, address)
+                        gpg.export_keys(key['fingerprint'], secret=secret),
+                        address)
+                    d = self.put_key(openpgp_key)
                     deferreds.append(d)
                 yield defer.gatherResults(deferreds)
 
@@ -368,10 +369,10 @@ class OpenPGPScheme(EncryptionScheme):
             if keydoc is None:
                 raise errors.KeyNotFound(address)
             leap_assert(
-                address in keydoc.content[KEY_ADDRESS_KEY],
+                address in keydoc.content[KEY_UIDS_KEY],
                 'Wrong address in key %s. Expected %s, found %s.'
                 % (keydoc.content[KEY_FINGERPRINT_KEY], address,
-                   keydoc.content[KEY_ADDRESS_KEY]))
+                   keydoc.content[KEY_UIDS_KEY]))
             key = build_key_from_dict(OpenPGPKey, keydoc.content,
                                       activedoc.content)
             key._gpgbinary = self._gpgbinary
@@ -381,13 +382,15 @@ class OpenPGPScheme(EncryptionScheme):
         d.addCallback(build_key)
         return d
 
-    def parse_ascii_key(self, key_data):
+    def parse_ascii_key(self, key_data, address=None):
         """
         Parses an ascii armored key (or key pair) data and returns
         the OpenPGPKey keys.
 
         :param key_data: the key data to be parsed.
         :type key_data: str or unicode
+        :param address: Active address for the key.
+        :type address: str
 
         :returns: the public key and private key (if applies) for that data.
         :rtype: (public, private) -> tuple(OpenPGPKey, OpenPGPKey)
@@ -408,12 +411,13 @@ class OpenPGPScheme(EncryptionScheme):
         openpgp_privkey = None
         if privkey:
             # build private key
-            openpgp_privkey = self._build_key_from_gpg(priv_info, privkey)
+            openpgp_privkey = self._build_key_from_gpg(priv_info, privkey,
+                                                       address)
             leap_check(pub_info['fingerprint'] == priv_info['fingerprint'],
                        'Fingerprints for public and private key differ.',
                        errors.KeyFingerprintMismatch)
         # build public key
-        openpgp_pubkey = self._build_key_from_gpg(pub_info, pubkey)
+        openpgp_pubkey = self._build_key_from_gpg(pub_info, pubkey, address)
 
         return (openpgp_pubkey, openpgp_privkey)
 
@@ -433,12 +437,13 @@ class OpenPGPScheme(EncryptionScheme):
 
         openpgp_privkey = None
         try:
-            openpgp_pubkey, openpgp_privkey = self.parse_ascii_key(key_data)
+            openpgp_pubkey, openpgp_privkey = self.parse_ascii_key(
+                key_data, address)
         except (errors.KeyAddressMismatch, errors.KeyFingerprintMismatch) as e:
             return defer.fail(e)
 
         def put_key(_, key):
-            return self.put_key(key, address)
+            return self.put_key(key)
 
         d = defer.succeed(None)
         if openpgp_pubkey is not None:
@@ -447,14 +452,12 @@ class OpenPGPScheme(EncryptionScheme):
             d.addCallback(put_key, openpgp_privkey)
         return d
 
-    def put_key(self, key, address):
+    def put_key(self, key):
         """
         Put C{key} in local storage.
 
         :param key: The key to be stored.
         :type key: OpenPGPKey
-        :param address: address for which this key will be active.
-        :type address: str
 
         :return: A Deferred which fires when the key is in the storage.
         :rtype: Deferred
@@ -471,31 +474,36 @@ class OpenPGPScheme(EncryptionScheme):
 
             key.merge(oldkey)
             keydoc.set_json(key.get_json())
-            deferred_key = self._soledad.put_doc(keydoc)
-
-            active_json = key.get_active_json(address)
-            if activedoc:
-                activedoc.set_json(active_json)
-                deferred_active = self._soledad.put_doc(activedoc)
-            else:
-                deferred_active = self._soledad.create_doc_from_json(
-                    active_json)
-
-            return defer.gatherResults([deferred_key, deferred_active])
+            d = self._soledad.put_doc(keydoc)
+            d.addCallback(put_active, activedoc)
+            return d
 
         def put_new_key(activedoc):
             deferreds = []
             if activedoc:
                 d = self._soledad.delete_doc(activedoc)
                 deferreds.append(d)
-            for json in [key.get_json(), key.get_active_json(address)]:
+            for json in [key.get_json(), key.get_active_json()]:
                 d = self._soledad.create_doc_from_json(json)
                 deferreds.append(d)
             return defer.gatherResults(deferreds)
 
-        dk = self._get_key_doc_from_fingerprint(key.fingerprint, key.private)
-        da = self._get_active_doc_from_address(address, key.private)
-        d = defer.gatherResults([dk, da])
+        def put_active(_, activedoc):
+            active_json = key.get_active_json()
+            if activedoc:
+                activedoc.set_json(active_json)
+                d = self._soledad.put_doc(activedoc)
+            else:
+                d = self._soledad.create_doc_from_json(active_json)
+            return d
+
+        def get_active_doc(keydoc):
+            d = self._get_active_doc_from_address(key.address, key.private)
+            d.addCallback(lambda activedoc: (keydoc, activedoc))
+            return d
+
+        d = self._get_key_doc_from_fingerprint(key.fingerprint, key.private)
+        d.addCallback(get_active_doc)
         d.addCallback(merge_and_put)
         return d
 
@@ -533,7 +541,7 @@ class OpenPGPScheme(EncryptionScheme):
         d.addCallback(get_key_from_active_doc)
         return d
 
-    def _build_key_from_gpg(self, key, key_data):
+    def _build_key_from_gpg(self, key, key_data, address=None):
         """
         Build an OpenPGPKey for C{address} based on C{key} from
         local gpg storage.
@@ -541,6 +549,8 @@ class OpenPGPScheme(EncryptionScheme):
         ASCII armored GPG key data has to be queried independently in this
         wrapper, so we receive it in C{key_data}.
 
+        :param address: Active address for the key.
+        :type address: str
         :param key: Key obtained from GPG storage.
         :type key: dict
         :param key_data: Key data obtained from GPG storage.
@@ -548,7 +558,7 @@ class OpenPGPScheme(EncryptionScheme):
         :return: An instance of the key.
         :rtype: OpenPGPKey
         """
-        return build_gpg_key(key, key_data, self._gpgbinary)
+        return build_gpg_key(key, key_data, address, self._gpgbinary)
 
     def delete_key(self, key):
         """
@@ -852,16 +862,20 @@ def process_ascii_key(key_data, gpgbinary, secret=False):
     return info, key
 
 
-def build_gpg_key(key_info, key_data, gpgbinary=None):
+def build_gpg_key(key_info, key_data, address=None, gpgbinary=None):
     expiry_date = None
     if key_info['expires']:
         expiry_date = datetime.fromtimestamp(int(key_info['expires']))
-    address = []
+    uids = []
     for uid in key_info['uids']:
-        address.append(_parse_address(uid))
+        uids.append(_parse_address(uid))
+    if address and address not in uids:
+        raise errors.KeyAddressMismatch("UIDs %s found, but expected %s"
+                                        % (str(uids), address))
 
     return OpenPGPKey(
-        address,
+        address=address,
+        uids=uids,
         gpgbinary=gpgbinary,
         fingerprint=key_info['fingerprint'],
         key_data=key_data,
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index 2fe9e4c..6347d56 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -76,7 +76,7 @@ class KeyManagerUtilTestCase(unittest.TestCase):
 
     def test_build_key_from_dict(self):
         kdict = {
-            'address': [ADDRESS],
+            'uids': [ADDRESS],
             'fingerprint': KEY_FINGERPRINT,
             'key_data': PUBLIC_KEY,
             'private': False,
@@ -94,7 +94,7 @@ class KeyManagerUtilTestCase(unittest.TestCase):
         }
         key = build_key_from_dict(OpenPGPKey, kdict, adict)
         self.assertEqual(
-            kdict['address'], key.address,
+            kdict['uids'], key.uids,
             'Wrong data in key.')
         self.assertEqual(
             kdict['fingerprint'], key.fingerprint,
@@ -117,6 +117,9 @@ class KeyManagerUtilTestCase(unittest.TestCase):
         self.assertEqual(
             datetime.fromtimestamp(kdict['refreshed_at']), key.refreshed_at,
             'Wrong data in key.')
+        self.assertEqual(
+            adict['address'], key.address,
+            'Wrong data in key.')
         self.assertEqual(
             ValidationLevels.get(adict['validation']), key.validation,
             'Wrong data in key.')
@@ -137,12 +140,12 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         # get public keys
         keys = yield km.get_all_keys(False)
         self.assertEqual(len(keys), 1, 'Wrong number of keys')
-        self.assertTrue(ADDRESS in keys[0].address)
+        self.assertTrue(ADDRESS in keys[0].uids)
         self.assertFalse(keys[0].private)
         # get private keys
         keys = yield km.get_all_keys(True)
         self.assertEqual(len(keys), 1, 'Wrong number of keys')
-        self.assertTrue(ADDRESS in keys[0].address)
+        self.assertTrue(ADDRESS in keys[0].uids)
         self.assertTrue(keys[0].private)
 
     @defer.inlineCallbacks
@@ -153,7 +156,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         key = yield km.get_key(ADDRESS, OpenPGPKey, private=False,
                                fetch_remote=False)
         self.assertTrue(key is not None)
-        self.assertTrue(ADDRESS in key.address)
+        self.assertTrue(ADDRESS in key.uids)
         self.assertEqual(
             key.fingerprint.lower(), KEY_FINGERPRINT.lower())
         self.assertFalse(key.private)
@@ -166,7 +169,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         key = yield km.get_key(ADDRESS, OpenPGPKey, private=True,
                                fetch_remote=False)
         self.assertTrue(key is not None)
-        self.assertTrue(ADDRESS in key.address)
+        self.assertTrue(ADDRESS in key.uids)
         self.assertEqual(
             key.fingerprint.lower(), KEY_FINGERPRINT.lower())
         self.assertTrue(key.private)
@@ -231,7 +234,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         key = yield self._fetch_key(km, ADDRESS, PUBLIC_KEY)
         self.assertIsInstance(key, OpenPGPKey)
-        self.assertTrue(ADDRESS in key.address)
+        self.assertTrue(ADDRESS in key.uids)
         self.assertEqual(key.validation, ValidationLevels.Provider_Trust)
 
     @defer.inlineCallbacks
@@ -243,7 +246,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         key = yield self._fetch_key(km, ADDRESS_OTHER, PUBLIC_KEY_OTHER)
         self.assertIsInstance(key, OpenPGPKey)
-        self.assertTrue(ADDRESS_OTHER in key.address)
+        self.assertTrue(ADDRESS_OTHER in key.uids)
         self.assertEqual(key.validation, ValidationLevels.Weak_Chain)
 
     def _fetch_key(self, km, address, key):
@@ -273,7 +276,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         yield km.put_raw_key(PUBLIC_KEY, OpenPGPKey, ADDRESS)
         key = yield km.get_key(ADDRESS, OpenPGPKey)
         self.assertIsInstance(key, OpenPGPKey)
-        self.assertTrue(ADDRESS in key.address)
+        self.assertTrue(ADDRESS in key.uids)
 
     @defer.inlineCallbacks
     def test_fetch_uri_ascii_key(self):
diff --git a/src/leap/keymanager/tests/test_openpgp.py b/src/leap/keymanager/tests/test_openpgp.py
index 8ed049f..0e5f6be 100644
--- a/src/leap/keymanager/tests/test_openpgp.py
+++ b/src/leap/keymanager/tests/test_openpgp.py
@@ -335,7 +335,7 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
                 deferreds = []
                 for k in (k1, k2, k3, k4, k5):
                     d = self._soledad.create_doc_from_json(
-                        k.get_active_json(ADDRESS))
+                        k.get_active_json())
                     deferreds.append(d)
                 return gatherResults(deferreds)
             elif args[0] == TYPE_FINGERPRINT_PRIVATE_INDEX:
-- 
cgit v1.2.3


From fbbed43d330b4e2a4499be086a857133ad93de97 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Mon, 21 Dec 2015 15:59:58 +0100
Subject: [doc] soledad docs fields

- Resolves: #7712
---
 changes/next-changelog.txt |  1 +
 docs/soledad-documents.rst | 77 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 docs/soledad-documents.rst

diff --git a/changes/next-changelog.txt b/changes/next-changelog.txt
index be6da72..163c1ee 100644
--- a/changes/next-changelog.txt
+++ b/changes/next-changelog.txt
@@ -13,6 +13,7 @@ Features
 - `#7485 <https://leap.se/code/issues/7485>`_: Move validation, usage and audited date to the active document.
 - `#7713 <https://leap.se/code/issues/7713>`_: Update soledad documents by adding versioning field.
 - `#7500 <https://leap.se/code/issues/7500>`_: Use fingerprints instead of key ids.
+- `#7712 <https://leap.se/code/issues/7712>`_: Document the soledad docs fields.
 - Make EncryptionKey aware of the active address.
 
 - `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
diff --git a/docs/soledad-documents.rst b/docs/soledad-documents.rst
new file mode 100644
index 0000000..67055b2
--- /dev/null
+++ b/docs/soledad-documents.rst
@@ -0,0 +1,77 @@
+=================
+Soledad Documents
+=================
+
+KeyManager uses two types of documents for the keyring:
+
+* key document, that stores each gpg key.
+
+* active document, that relates an address to its corresponding key.
+
+
+Each key can have 0 or more active documents with a different email address
+each:
+
+::
+
+  .-------------.          .-------------.
+  | foo@foo.com |          | bar@bar.com |
+  '-------------'          '-------------'
+         |                        |     
+         |      .-----------.     |     
+         |      |           |     |     
+         |      |    key    |     |     
+         '----->|           |<----'
+                |           |     
+                '-----------'
+
+
+Fields in a key document:
+
+* uids
+
+* fingerprint
+
+* key_data
+
+* private. bool marking if the key is private or public
+
+* length
+
+* expiry_date
+
+* refreshed_at
+
+* version = 1
+
+* type = "OpenPGPKey"
+
+* tags = ["keymanager-key"]
+
+
+Fields in an active document:
+
+* address
+
+* fingerprint
+
+* private
+
+* validation
+
+* last_audited_at
+
+* encr_used
+
+* sign_used
+
+* version = 1
+
+* type = "OpenPGPKey-active"
+
+* tags = ["keymanager-active"]
+
+
+The meaning of validation, encr_used and sign_used is related to the `Transitional Key Validation`_
+
+.. _Transitional Key Validation: https://leap.se/en/docs/design/transitional-key-validation
-- 
cgit v1.2.3


From 1b4d08537e2a526e8f5a76dbe7c7f6d979025296 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Thu, 21 Jan 2016 18:48:23 +0100
Subject: [feat] update usage only if needed

During encryption we where updating 'enc_used' in the key without
checking if it was already set.
---
 src/leap/keymanager/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 99ee163..9e3b6ee 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -579,8 +579,9 @@ class KeyManager(object):
             encrypted = yield _keys.encrypt(
                 data, pubkey, passphrase, sign=signkey,
                 cipher_algo=cipher_algo)
-            pubkey.encr_used = True
-            yield _keys.put_key(pubkey)
+            if not pubkey.encr_used:
+                pubkey.encr_used = True
+                yield _keys.put_key(pubkey)
             defer.returnValue(encrypted)
 
         dpub = self.get_key(address, ktype, private=False,
-- 
cgit v1.2.3


From d44d72c816ebc668931071cf5bd8e37e0be32e05 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Thu, 25 Feb 2016 11:34:45 -0600
Subject: [test] refactor key deletion tests

---
 src/leap/keymanager/tests/__init__.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/leap/keymanager/tests/__init__.py b/src/leap/keymanager/tests/__init__.py
index 4fbf63e..20d05e8 100644
--- a/src/leap/keymanager/tests/__init__.py
+++ b/src/leap/keymanager/tests/__init__.py
@@ -54,6 +54,14 @@ class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
     def tearDown(self):
         km = self._key_manager()
 
+        # wait for the indexes to be ready for the tear down
+        d = km._wrapper_map[OpenPGPKey].deferred_init
+        d.addCallback(lambda _: self.delete_all_keys(km))
+        d.addCallback(lambda _: self.tearDownEnv())
+        d.addCallback(lambda _: self._soledad.close())
+        return d
+
+    def delete_all_keys(self, km):
         def delete_keys(keys):
             deferreds = []
             for key in keys:
@@ -61,26 +69,18 @@ class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
                 deferreds.append(d)
             return gatherResults(deferreds)
 
-        def get_and_delete_keys(_):
-            deferreds = []
-            for private in [True, False]:
-                d = km.get_all_keys(private=private)
-                d.addCallback(delete_keys)
-                d.addCallback(check_deleted, private)
-                deferreds.append(d)
-            return gatherResults(deferreds)
-
         def check_deleted(_, private):
             d = km.get_all_keys(private=private)
             d.addCallback(lambda keys: self.assertEqual(keys, []))
             return d
 
-        # wait for the indexes to be ready for the tear down
-        d = km._wrapper_map[OpenPGPKey].deferred_init
-        d.addCallback(get_and_delete_keys)
-        d.addCallback(lambda _: self.tearDownEnv())
-        d.addCallback(lambda _: self._soledad.close())
-        return d
+        deferreds = []
+        for private in [True, False]:
+            d = km.get_all_keys(private=private)
+            d.addCallback(delete_keys)
+            d.addCallback(check_deleted, private)
+            deferreds.append(d)
+        return gatherResults(deferreds)
 
     def _key_manager(self, user=ADDRESS, url='', token=None,
                      ca_cert_path=None):
-- 
cgit v1.2.3


From 288c775034d4c18846518a11677e4580a91cf437 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Sun, 20 Mar 2016 18:44:23 +0100
Subject: [bug] Return KeyNotFound Failure if not valid key is given to
 put_raw_key

- Resolves: #7974
---
 changes/next-changelog.txt      |  2 ++
 src/leap/keymanager/__init__.py | 11 ++++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/changes/next-changelog.txt b/changes/next-changelog.txt
index 163c1ee..2dd34d4 100644
--- a/changes/next-changelog.txt
+++ b/changes/next-changelog.txt
@@ -21,6 +21,8 @@ Features
 
 Bugfixes
 ~~~~~~~~
+- `#7974 <https://leap.se/code/issues/7974>`_: Return KeyNotFound Failure if not valid key is given to put_raw_key.
+
 - `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235.
 - Bugfix without related issue number.
 
diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 9e3b6ee..1106c23 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -820,9 +820,10 @@ class KeyManager(object):
 
         :return: A Deferred which fires when the key is in the storage, or
                  which fails with KeyAddressMismatch if address doesn't match
-                 any uid on the key or fails with KeyNotValidUpdate if a key
-                 with the same uid exists and the new one is not a valid update
-                 for it.
+                 any uid on the key or fails with KeyNotFound if no OpenPGP
+                 material was found in key or fails with KeyNotValidUpdate if a
+                 key with the same uid exists and the new one is not a valid
+                 update for it.
         :rtype: Deferred
 
         :raise UnsupportedKeyTypeError: if invalid key type
@@ -831,6 +832,10 @@ class KeyManager(object):
         _keys = self._wrapper_map[ktype]
 
         pubkey, privkey = _keys.parse_ascii_key(key, address)
+
+        if pubkey is None:
+            return defer.fail(KeyNotFound(key))
+
         pubkey.validation = validation
         d = self.put_key(pubkey)
         if privkey is not None:
-- 
cgit v1.2.3


From 495efc574a88d33c5528a1211faf8ca1e684773e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 1 Apr 2016 17:25:58 -0400
Subject: [pkg] update to versioneer 0.16

---
 MANIFEST.in                     |    1 +
 setup.cfg                       |    7 +
 setup.py                        |   51 +-
 src/leap/keymanager/__init__.py |    4 +
 src/leap/keymanager/_version.py |  544 ++++++++---
 versioneer.py                   | 2053 ++++++++++++++++++++++++++++++---------
 6 files changed, 2026 insertions(+), 634 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 7f6148e..d1a0e9a 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,3 +2,4 @@ include pkg/*
 include versioneer.py
 include LICENSE
 include CHANGELOG
+include src/leap/keymanager/_version.py
diff --git a/setup.cfg b/setup.cfg
index 51070c6..4a24b89 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -8,3 +8,10 @@ ignore = E731
 [flake8]
 exclude = versioneer.py,_version.py,*.egg,build,docs
 ignore = E731
+
+[versioneer]
+VCS = git
+style = pep440
+versionfile_source = src/leap/keymanager/_version.py
+versionfile_build = leap/keymanager/_version.py
+tag_prefix = 
diff --git a/setup.py b/setup.py
index 26840c6..4a05a1e 100644
--- a/setup.py
+++ b/setup.py
@@ -24,12 +24,7 @@ from setuptools import Command
 
 from pkg import utils
 
-
 import versioneer
-versioneer.versionfile_source = 'src/leap/keymanager/_version.py'
-versioneer.versionfile_build = 'leap/keymanager/_version.py'
-versioneer.tag_prefix = ''  # tags are like 1.2.0
-versioneer.parentdir_prefix = 'leap.keymanager-'
 
 trove_classifiers = [
     'Development Status :: 4 - Beta',
@@ -45,11 +40,11 @@ trove_classifiers = [
     'Topic :: Software Development :: Libraries',
 ]
 
-DOWNLOAD_BASE = ('https://github.com/leapcode/keymanager/'
+DOWNLOAD_BASE = ('https://github.com/leapcode/bitmask_client/'
                  'archive/%s.tar.gz')
 _versions = versioneer.get_versions()
 VERSION = _versions['version']
-VERSION_FULL = _versions['full']
+VERSION_REVISION = _versions['full-revisionid']
 DOWNLOAD_URL = ""
 
 # get the short version for the download url
@@ -58,15 +53,30 @@ if len(_version_short) > 0:
     VERSION_SHORT = _version_short[0]
     DOWNLOAD_URL = DOWNLOAD_BASE % VERSION_SHORT
 
-cmdclass = versioneer.get_cmdclass()
-
 
 class freeze_debianver(Command):
+
     """
     Freezes the version in a debian branch.
     To be used after merging the development branch onto the debian one.
     """
     user_options = []
+    template = r"""
+# This file was generated by the `freeze_debianver` command in setup.py
+# Using 'versioneer.py' (0.16) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+version_version = '{version}'
+full_revisionid = '{full_revisionid}'
+"""
+    templatefun = r"""
+
+def get_versions(default={}, verbose=False):
+        return {'version': version_version,
+                'full-revisionid': full_revisionid}
+"""
 
     def initialize_options(self):
         pass
@@ -80,28 +90,15 @@ class freeze_debianver(Command):
         if proceed != "y":
             print("He. You scared. Aborting.")
             return
-        template = r"""
-# This file was generated by the `freeze_debianver` command in setup.py
-# Using 'versioneer.py' (0.7+) from
-# revision-control system data, or from the parent directory name of an
-# unpacked source archive. Distribution tarballs contain a pre-generated copy
-# of this file.
-
-version_version = '{version}'
-version_full = '{version_full}'
-"""
-        templatefun = r"""
-
-def get_versions(default={}, verbose=False):
-        return {'version': version_version, 'full': version_full}
-"""
-        subst_template = template.format(
+        subst_template = self.template.format(
             version=VERSION_SHORT,
-            version_full=VERSION_FULL) + templatefun
-        with open(versioneer.versionfile_source, 'w') as f:
+            full_revisionid=VERSION_REVISION) + self.templatefun
+        versioneer_cfg = versioneer.get_config_from_root('.')
+        with open(versioneer_cfg.versionfile_source, 'w') as f:
             f.write(subst_template)
 
 
+cmdclass = versioneer.get_cmdclass()
 cmdclass["freeze_debianver"] = freeze_debianver
 
 # XXX add ref to docs
diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 1106c23..91cd2f8 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -921,3 +921,7 @@ def _get_domain(url):
     :rtype: str
     """
     return urlparse(url).hostname
+
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions
diff --git a/src/leap/keymanager/_version.py b/src/leap/keymanager/_version.py
index 5153a9b..b28c697 100644
--- a/src/leap/keymanager/_version.py
+++ b/src/leap/keymanager/_version.py
@@ -1,73 +1,157 @@
+
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
+
+"""Git implementation of _version.py."""
 
-# these strings will be replaced by git during git-archive
+import errno
+import os
+import re
 import subprocess
 import sys
-import re
-import os.path
 
-IN_LONG_VERSION_PY = True
 
-git_refnames = "$Format:%d$"
-git_full = "$Format:%H$"
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "$Format:%d$"
+    git_full = "$Format:%H$"
+    keywords = {"refnames": git_refnames, "full": git_full}
+    return keywords
 
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.cmd on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "pep440"
+    cfg.tag_prefix = ""
+    cfg.parentdir_prefix = "None"
+    cfg.versionfile_source = "src/leap/keymanager/_version.py"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None
+    else:
         if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
+            print("unable to find command, tried %s" % (commands,))
         return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %s (error)" % args[0])
+            print("unable to run %s (error)" % dispcmd)
         return None
     return stdout
 
 
-def get_expanded_variables(versionfile_source):
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with "
+                  "prefix '%s'" % (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source, "r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {}  # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -83,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
         # "stabilization", as well as "HEAD" and "master".
         tags = set([r for r in refs if re.search(r'\d', r)])
         if verbose:
-            print("discarding '%s', no digits" % ",".join(refs - tags))
+            print("discarding '%s', no digits" % ",".join(refs-tags))
     if verbose:
         print("likely tags: %s" % ",".join(sorted(tags)))
     for ref in sorted(tags):
@@ -93,114 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             if verbose:
                 print("picking %s" % r)
             return {"version": r,
-                    "full": variables["full"].strip()}
-    # no suitable tags, so we use the full revision id
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return {"version": variables["full"].strip(),
-            "full": variables["full"].strip()}
-
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {}  # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
     if not os.path.exists(os.path.join(root, ".git")):
         if verbose:
             print("no .git in %s" % root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.cmd"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" %
-                  (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source,
-                            verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {}  # without __file__, we have no hope
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%s*" % tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
+
+    try:
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
+
+    try:
+        root = os.path.realpath(__file__)
         # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
-    else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree"}
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start "
-                  "with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = ""
-parentdir_prefix = "leap.keymanager-"
-versionfile_source = "src/leap/keymanager/_version.py"
-
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = {"refnames": git_refnames, "full": git_full}
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version"}
diff --git a/versioneer.py b/versioneer.py
index 4e2c0a5..7ed2a21 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -1,170 +1,641 @@
-#! /usr/bin/python
 
-"""versioneer.py
+# Version: 0.16
 
-(like a rocketeer, but for versions)
+"""The Versioneer - like a rocketeer, but for versions.
 
+The Versioneer
+==============
+
+* like a rocketeer, but for versions!
 * https://github.com/warner/python-versioneer
 * Brian Warner
 * License: Public Domain
-* Version: 0.7+
-
-This file helps distutils-based projects manage their version number by just
-creating version-control tags.
-
-For developers who work from a VCS-generated tree (e.g. 'git clone' etc),
-each 'setup.py version', 'setup.py build', 'setup.py sdist' will compute a
-version number by asking your version-control tool about the current
-checkout. The version number will be written into a generated _version.py
-file of your choosing, where it can be included by your __init__.py
-
-For users who work from a VCS-generated tarball (e.g. 'git archive'), it will
-compute a version number by looking at the name of the directory created when
-te tarball is unpacked. This conventionally includes both the name of the
-project and a version number.
-
-For users who work from a tarball built by 'setup.py sdist', it will get a
-version number from a previously-generated _version.py file.
-
-As a result, loading code directly from the source tree will not result in a
-real version. If you want real versions from VCS trees (where you frequently
-update from the upstream repository, or do new development), you will need to
-do a 'setup.py version' after each update, and load code from the build/
-directory.
-
-You need to provide this code with a few configuration values:
-
- versionfile_source:
-    A project-relative pathname into which the generated version strings
-    should be written. This is usually a _version.py next to your project's
-    main __init__.py file. If your project uses src/myproject/__init__.py,
-    this should be 'src/myproject/_version.py'. This file should be checked
-    in to your VCS as usual: the copy created below by 'setup.py
-    update_files' will include code that parses expanded VCS keywords in
-    generated tarballs. The 'build' and 'sdist' commands will replace it with
-    a copy that has just the calculated version string.
-
- versionfile_build:
-    Like versionfile_source, but relative to the build directory instead of
-    the source directory. These will differ when your setup.py uses
-    'package_dir='. If you have package_dir={'myproject': 'src/myproject'},
-    then you will probably have versionfile_build='myproject/_version.py' and
-    versionfile_source='src/myproject/_version.py'.
-
- tag_prefix: a string, like 'PROJECTNAME-', which appears at the start of all
-             VCS tags. If your tags look like 'myproject-1.2.0', then you
-             should use tag_prefix='myproject-'. If you use unprefixed tags
-             like '1.2.0', this should be an empty string.
-
- parentdir_prefix: a string, frequently the same as tag_prefix, which
-                   appears at the start of all unpacked tarball filenames. If
-                   your tarball unpacks into 'myproject-1.2.0', this should
-                   be 'myproject-'.
-
-To use it:
-
- 1: include this file in the top level of your project
- 2: make the following changes to the top of your setup.py:
-     import versioneer
-     versioneer.versionfile_source = 'src/myproject/_version.py'
-     versioneer.versionfile_build = 'myproject/_version.py'
-     versioneer.tag_prefix = '' # tags are like 1.2.0
-     versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0'
- 3: add the following arguments to the setup() call in your setup.py:
-     version=versioneer.get_version(),
-     cmdclass=versioneer.get_cmdclass(),
- 4: run 'setup.py update_files', which will create _version.py, and will
-    modify your __init__.py to define __version__ (by calling a function
-    from _version.py)
- 5: modify your MANIFEST.in to include versioneer.py
- 6: add both versioneer.py and the generated _version.py to your VCS
-"""
+* Compatible With: python2.6, 2.7, 3.3, 3.4, 3.5, and pypy
+* [![Latest Version]
+(https://pypip.in/version/versioneer/badge.svg?style=flat)
+](https://pypi.python.org/pypi/versioneer/)
+* [![Build Status]
+(https://travis-ci.org/warner/python-versioneer.png?branch=master)
+](https://travis-ci.org/warner/python-versioneer)
+
+This is a tool for managing a recorded version number in distutils-based
+python projects. The goal is to remove the tedious and error-prone "update
+the embedded version string" step from your release process. Making a new
+release should be as easy as recording a new tag in your version-control
+system, and maybe making new tarballs.
+
+
+## Quick Install
+
+* `pip install versioneer` to somewhere to your $PATH
+* add a `[versioneer]` section to your setup.cfg (see below)
+* run `versioneer install` in your source tree, commit the results
+
+## Version Identifiers
+
+Source trees come from a variety of places:
+
+* a version-control system checkout (mostly used by developers)
+* a nightly tarball, produced by build automation
+* a snapshot tarball, produced by a web-based VCS browser, like github's
+  "tarball from tag" feature
+* a release tarball, produced by "setup.py sdist", distributed through PyPI
+
+Within each source tree, the version identifier (either a string or a number,
+this tool is format-agnostic) can come from a variety of places:
+
+* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows
+  about recent "tags" and an absolute revision-id
+* the name of the directory into which the tarball was unpacked
+* an expanded VCS keyword ($Id$, etc)
+* a `_version.py` created by some earlier build step
+
+For released software, the version identifier is closely related to a VCS
+tag. Some projects use tag names that include more than just the version
+string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool
+needs to strip the tag prefix to extract the version identifier. For
+unreleased software (between tags), the version identifier should provide
+enough information to help developers recreate the same tree, while also
+giving them an idea of roughly how old the tree is (after version 1.2, before
+version 1.3). Many VCS systems can report a description that captures this,
+for example `git describe --tags --dirty --always` reports things like
+"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the
+0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has
+uncommitted changes.
+
+The version identifier is used for multiple purposes:
+
+* to allow the module to self-identify its version: `myproject.__version__`
+* to choose a name and prefix for a 'setup.py sdist' tarball
+
+## Theory of Operation
+
+Versioneer works by adding a special `_version.py` file into your source
+tree, where your `__init__.py` can import it. This `_version.py` knows how to
+dynamically ask the VCS tool for version information at import time.
+
+`_version.py` also contains `$Revision$` markers, and the installation
+process marks `_version.py` to have this marker rewritten with a tag name
+during the `git archive` command. As a result, generated tarballs will
+contain enough information to get the proper version.
+
+To allow `setup.py` to compute a version too, a `versioneer.py` is added to
+the top level of your source tree, next to `setup.py` and the `setup.cfg`
+that configures it. This overrides several distutils/setuptools commands to
+compute the version when invoked, and changes `setup.py build` and `setup.py
+sdist` to replace `_version.py` with a small static file that contains just
+the generated version data.
+
+## Installation
+
+First, decide on values for the following configuration variables:
+
+* `VCS`: the version control system you use. Currently accepts "git".
+
+* `style`: the style of version string to be produced. See "Styles" below for
+  details. Defaults to "pep440", which looks like
+  `TAG[+DISTANCE.gSHORTHASH[.dirty]]`.
+
+* `versionfile_source`:
+
+  A project-relative pathname into which the generated version strings should
+  be written. This is usually a `_version.py` next to your project's main
+  `__init__.py` file, so it can be imported at runtime. If your project uses
+  `src/myproject/__init__.py`, this should be `src/myproject/_version.py`.
+  This file should be checked in to your VCS as usual: the copy created below
+  by `setup.py setup_versioneer` will include code that parses expanded VCS
+  keywords in generated tarballs. The 'build' and 'sdist' commands will
+  replace it with a copy that has just the calculated version string.
+
+  This must be set even if your project does not have any modules (and will
+  therefore never import `_version.py`), since "setup.py sdist" -based trees
+  still need somewhere to record the pre-calculated version strings. Anywhere
+  in the source tree should do. If there is a `__init__.py` next to your
+  `_version.py`, the `setup.py setup_versioneer` command (described below)
+  will append some `__version__`-setting assignments, if they aren't already
+  present.
+
+* `versionfile_build`:
+
+  Like `versionfile_source`, but relative to the build directory instead of
+  the source directory. These will differ when your setup.py uses
+  'package_dir='. If you have `package_dir={'myproject': 'src/myproject'}`,
+  then you will probably have `versionfile_build='myproject/_version.py'` and
+  `versionfile_source='src/myproject/_version.py'`.
+
+  If this is set to None, then `setup.py build` will not attempt to rewrite
+  any `_version.py` in the built tree. If your project does not have any
+  libraries (e.g. if it only builds a script), then you should use
+  `versionfile_build = None`. To actually use the computed version string,
+  your `setup.py` will need to override `distutils.command.build_scripts`
+  with a subclass that explicitly inserts a copy of
+  `versioneer.get_version()` into your script file. See
+  `test/demoapp-script-only/setup.py` for an example.
+
+* `tag_prefix`:
+
+  a string, like 'PROJECTNAME-', which appears at the start of all VCS tags.
+  If your tags look like 'myproject-1.2.0', then you should use
+  tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this
+  should be an empty string, using either `tag_prefix=` or `tag_prefix=''`.
+
+* `parentdir_prefix`:
+
+  a optional string, frequently the same as tag_prefix, which appears at the
+  start of all unpacked tarball filenames. If your tarball unpacks into
+  'myproject-1.2.0', this should be 'myproject-'. To disable this feature,
+  just omit the field from your `setup.cfg`.
+
+This tool provides one script, named `versioneer`. That script has one mode,
+"install", which writes a copy of `versioneer.py` into the current directory
+and runs `versioneer.py setup` to finish the installation.
+
+To versioneer-enable your project:
+
+* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and
+  populating it with the configuration values you decided earlier (note that
+  the option names are not case-sensitive):
+
+  ````
+  [versioneer]
+  VCS = git
+  style = pep440
+  versionfile_source = src/myproject/_version.py
+  versionfile_build = myproject/_version.py
+  tag_prefix =
+  parentdir_prefix = myproject-
+  ````
+
+* 2: Run `versioneer install`. This will do the following:
+
+  * copy `versioneer.py` into the top of your source tree
+  * create `_version.py` in the right place (`versionfile_source`)
+  * modify your `__init__.py` (if one exists next to `_version.py`) to define
+    `__version__` (by calling a function from `_version.py`)
+  * modify your `MANIFEST.in` to include both `versioneer.py` and the
+    generated `_version.py` in sdist tarballs
+
+  `versioneer install` will complain about any problems it finds with your
+  `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all
+  the problems.
+
+* 3: add a `import versioneer` to your setup.py, and add the following
+  arguments to the setup() call:
+
+        version=versioneer.get_version(),
+        cmdclass=versioneer.get_cmdclass(),
+
+* 4: commit these changes to your VCS. To make sure you won't forget,
+  `versioneer install` will mark everything it touched for addition using
+  `git add`. Don't forget to add `setup.py` and `setup.cfg` too.
+
+## Post-Installation Usage
+
+Once established, all uses of your tree from a VCS checkout should get the
+current version string. All generated tarballs should include an embedded
+version string (so users who unpack them will not need a VCS tool installed).
+
+If you distribute your project through PyPI, then the release process should
+boil down to two steps:
+
+* 1: git tag 1.0
+* 2: python setup.py register sdist upload
+
+If you distribute it through github (i.e. users use github to generate
+tarballs with `git archive`), the process is:
+
+* 1: git tag 1.0
+* 2: git push; git push --tags
+
+Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at
+least one tag in its history.
+
+## Version-String Flavors
+
+Code which uses Versioneer can learn about its version string at runtime by
+importing `_version` from your main `__init__.py` file and running the
+`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can
+import the top-level `versioneer.py` and run `get_versions()`.
+
+Both functions return a dictionary with different flavors of version
+information:
+
+* `['version']`: A condensed version string, rendered using the selected
+  style. This is the most commonly used value for the project's version
+  string. The default "pep440" style yields strings like `0.11`,
+  `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section
+  below for alternative styles.
+
+* `['full-revisionid']`: detailed revision identifier. For Git, this is the
+  full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac".
+
+* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that
+  this is only accurate if run in a VCS checkout, otherwise it is likely to
+  be False or None
+
+* `['error']`: if the version string could not be computed, this will be set
+  to a string describing the problem, otherwise it will be None. It may be
+  useful to throw an exception in setup.py if this is set, to avoid e.g.
+  creating tarballs with a version string of "unknown".
+
+Some variants are more useful than others. Including `full-revisionid` in a
+bug report should allow developers to reconstruct the exact code being tested
+(or indicate the presence of local changes that should be shared with the
+developers). `version` is suitable for display in an "about" box or a CLI
+`--version` output: it can be easily compared against release notes and lists
+of bugs fixed in various releases.
+
+The installer adds the following text to your `__init__.py` to place a basic
+version in `YOURPROJECT.__version__`:
+
+    from ._version import get_versions
+    __version__ = get_versions()['version']
+    del get_versions
+
+## Styles
+
+The setup.cfg `style=` configuration controls how the VCS information is
+rendered into a version string.
+
+The default style, "pep440", produces a PEP440-compliant string, equal to the
+un-prefixed tag name for actual releases, and containing an additional "local
+version" section with more detail for in-between builds. For Git, this is
+TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags
+--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the
+tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and
+that this commit is two revisions ("+2") beyond the "0.11" tag. For released
+software (exactly equal to a known tag), the identifier will only contain the
+stripped tag, e.g. "0.11".
+
+Other styles are available. See details.md in the Versioneer source tree for
+descriptions.
+
+## Debugging
+
+Versioneer tries to avoid fatal errors: if something goes wrong, it will tend
+to return a version of "0+unknown". To investigate the problem, run `setup.py
+version`, which will run the version-lookup code in a verbose mode, and will
+display the full contents of `get_versions()` (including the `error` string,
+which may help identify what went wrong).
+
+## Updating Versioneer
+
+To upgrade your project to a new release of Versioneer, do the following:
+
+* install the new Versioneer (`pip install -U versioneer` or equivalent)
+* edit `setup.cfg`, if necessary, to include any new configuration settings
+  indicated by the release notes
+* re-run `versioneer install` in your source tree, to replace
+  `SRC/_version.py`
+* commit any changed files
+
+### Upgrading to 0.16
+
+Nothing special.
+
+### Upgrading to 0.15
+
+Starting with this version, Versioneer is configured with a `[versioneer]`
+section in your `setup.cfg` file. Earlier versions required the `setup.py` to
+set attributes on the `versioneer` module immediately after import. The new
+version will refuse to run (raising an exception during import) until you
+have provided the necessary `setup.cfg` section.
+
+In addition, the Versioneer package provides an executable named
+`versioneer`, and the installation process is driven by running `versioneer
+install`. In 0.14 and earlier, the executable was named
+`versioneer-installer` and was run without an argument.
+
+### Upgrading to 0.14
 
-import os, sys, re
-from distutils.core import Command
-from distutils.command.sdist import sdist as _sdist
-from distutils.command.build import build as _build
+0.14 changes the format of the version string. 0.13 and earlier used
+hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a
+plus-separated "local version" section strings, with dot-separated
+components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old
+format, but should be ok with the new one.
 
-versionfile_source = None
-versionfile_build = None
-tag_prefix = None
-parentdir_prefix = None
+### Upgrading from 0.11 to 0.12
 
-VCS = "git"
-IN_LONG_VERSION_PY = False
+Nothing special.
 
+### Upgrading from 0.10 to 0.11
 
-LONG_VERSION_PY = '''
-IN_LONG_VERSION_PY = True
+You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running
+`setup.py setup_versioneer`. This will enable the use of additional
+version-control systems (SVN, etc) in the future.
+
+## Future Directions
+
+This tool is designed to make it easily extended to other version-control
+systems: all VCS-specific components are in separate directories like
+src/git/ . The top-level `versioneer.py` script is assembled from these
+components by running make-versioneer.py . In the future, make-versioneer.py
+will take a VCS name as an argument, and will construct a version of
+`versioneer.py` that is specific to the given VCS. It might also take the
+configuration arguments that are currently provided manually during
+installation by editing setup.py . Alternatively, it might go the other
+direction and include code from all supported VCS systems, reducing the
+number of intermediate scripts.
+
+
+## License
+
+To make Versioneer easier to embed, all its code is dedicated to the public
+domain. The `_version.py` that it creates is also in the public domain.
+Specifically, both are released under the Creative Commons "Public Domain
+Dedication" license (CC0-1.0), as described in
+https://creativecommons.org/publicdomain/zero/1.0/ .
+
+"""
+
+from __future__ import print_function
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+import errno
+import json
+import os
+import re
+import subprocess
+import sys
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_root():
+    """Get the project root directory.
+
+    We require that all commands are run from the project root, i.e. the
+    directory that contains setup.py, setup.cfg, and versioneer.py .
+    """
+    root = os.path.realpath(os.path.abspath(os.getcwd()))
+    setup_py = os.path.join(root, "setup.py")
+    versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        # allow 'python path/to/setup.py COMMAND'
+        root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
+        setup_py = os.path.join(root, "setup.py")
+        versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        err = ("Versioneer was unable to run the project root directory. "
+               "Versioneer requires setup.py to be executed from "
+               "its immediate directory (like 'python setup.py COMMAND'), "
+               "or in a way that lets it use sys.argv[0] to find the root "
+               "(like 'python path/to/setup.py COMMAND').")
+        raise VersioneerBadRootError(err)
+    try:
+        # Certain runtime workflows (setup.py install/develop in a setuptools
+        # tree) execute all dependencies in a single python process, so
+        # "versioneer" may be imported multiple times, and python's shared
+        # module-import table will cache the first one. So we can't use
+        # os.path.dirname(__file__), as that will find whichever
+        # versioneer.py was first imported, even in later projects.
+        me = os.path.realpath(os.path.abspath(__file__))
+        if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]:
+            print("Warning: build in %s is using versioneer.py from %s"
+                  % (os.path.dirname(me), versioneer_py))
+    except NameError:
+        pass
+    return root
+
+
+def get_config_from_root(root):
+    """Read the project setup.cfg file to determine Versioneer config."""
+    # This might raise EnvironmentError (if setup.cfg is missing), or
+    # configparser.NoSectionError (if it lacks a [versioneer] section), or
+    # configparser.NoOptionError (if it lacks "VCS="). See the docstring at
+    # the top of versioneer.py for instructions on writing your setup.cfg .
+    setup_cfg = os.path.join(root, "setup.cfg")
+    parser = configparser.SafeConfigParser()
+    with open(setup_cfg, "r") as f:
+        parser.readfp(f)
+    VCS = parser.get("versioneer", "VCS")  # mandatory
+
+    def get(parser, name):
+        if parser.has_option("versioneer", name):
+            return parser.get("versioneer", name)
+        return None
+    cfg = VersioneerConfig()
+    cfg.VCS = VCS
+    cfg.style = get(parser, "style") or ""
+    cfg.versionfile_source = get(parser, "versionfile_source")
+    cfg.versionfile_build = get(parser, "versionfile_build")
+    cfg.tag_prefix = get(parser, "tag_prefix")
+    if cfg.tag_prefix in ("''", '""'):
+        cfg.tag_prefix = ""
+    cfg.parentdir_prefix = get(parser, "parentdir_prefix")
+    cfg.verbose = get(parser, "verbose")
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+# these dictionaries contain VCS-specific tools
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None
+    else:
+        if verbose:
+            print("unable to find command, tried %s" % (commands,))
+        return None
+    stdout = p.communicate()[0].strip()
+    if sys.version_info[0] >= 3:
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %s (error)" % dispcmd)
+        return None
+    return stdout
+LONG_VERSION_PY['git'] = '''
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
-
-# these strings will be replaced by git during git-archive
-git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
-git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
 
+"""Git implementation of _version.py."""
 
+import errno
+import os
+import re
 import subprocess
 import sys
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.exe on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
+    git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+    keywords = {"refnames": git_refnames, "full": git_full}
+    return keywords
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "%(STYLE)s"
+    cfg.tag_prefix = "%(TAG_PREFIX)s"
+    cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s"
+    cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %%s" %% dispcmd)
+                print(e)
+            return None
+    else:
         if verbose:
-            print("unable to run %%s" %% args[0])
-            print(e)
+            print("unable to find command, tried %%s" %% (commands,))
         return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %%s (error)" %% args[0])
+            print("unable to run %%s (error)" %% dispcmd)
         return None
     return stdout
 
 
-import sys
-import re
-import os.path
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%%s', but '%%s' doesn't start with "
+                  "prefix '%%s'" %% (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
 
-def get_expanded_variables(versionfile_source):
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source,"r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
+
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -189,172 +660,350 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %%s" %% r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
     if not os.path.exists(os.path.join(root, ".git")):
         if verbose:
             print("no .git in %%s" %% root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
-        # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%%s*" %% tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%%s'"
+                               %% describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%%s' doesn't start with prefix '%%s'"
+                print(fmt %% (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'"
+                               %% (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%%d" %% pieces["distance"]
     else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+        # exception #1
+        rendered = "0.post.dev%%d" %% pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%%s" %% pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%%s" %% pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %%
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = "%(TAG_PREFIX)s"
-parentdir_prefix = "%(PARENTDIR_PREFIX)s"
-versionfile_source = "%(VERSIONFILE_SOURCE)s"
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = { "refnames": git_refnames, "full": git_full }
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
 
-'''
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
 
+    Like 'git describe --tags --dirty --always'.
 
-import subprocess
-import sys
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%%s'" %% style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
 
-def run_command(args, cwd=None, verbose=False):
     try:
-        # remember shell=False, so use git.exe on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
-        if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
-        return None
-    stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
-        stdout = stdout.decode()
-    if p.returncode != 0:
-        if verbose:
-            print("unable to run %s (error)" % args[0])
-        return None
-    return stdout
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
 
+    try:
+        root = os.path.realpath(__file__)
+        # versionfile_source is the relative path from the top of the source
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
+            root = os.path.dirname(root)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree"}
 
-import sys
-import re
-import os.path
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
 
-def get_expanded_variables(versionfile_source):
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version"}
+'''
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source,"r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
+
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {} # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -379,107 +1028,122 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             r = ref[len(tag_prefix):]
             if verbose:
                 print("picking %s" % r)
-            return { "version": r,
-                     "full": variables["full"].strip() }
-    # no suitable tags, so we use the full revision id
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return { "version": variables["full"].strip(),
-             "full": variables["full"].strip() }
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {} # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
     if not os.path.exists(os.path.join(root, ".git")):
         if verbose:
             print("no .git in %s" % root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {} # without __file__, we have no hope
-        # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%s*" % tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
     else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
+    return pieces
 
-import sys
 
-def do_vcs_install(versionfile_source, ipy):
-    GIT = "git"
+def do_vcs_install(manifest_in, versionfile_source, ipy):
+    """Git-specific installation logic for Versioneer.
+
+    For Git, this means creating/changing .gitattributes to mark _version.py
+    for export-time keyword substitution.
+    """
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.exe"
-    run_command([GIT, "add", "versioneer.py"])
-    run_command([GIT, "add", versionfile_source])
-    run_command([GIT, "add", ipy])
+        GITS = ["git.cmd", "git.exe"]
+    files = [manifest_in, versionfile_source]
+    if ipy:
+        files.append(ipy)
+    try:
+        me = __file__
+        if me.endswith(".pyc") or me.endswith(".pyo"):
+            me = os.path.splitext(me)[0] + ".py"
+        versioneer_file = os.path.relpath(me)
+    except NameError:
+        versioneer_file = "versioneer.py"
+    files.append(versioneer_file)
     present = False
     try:
         f = open(".gitattributes", "r")
@@ -494,135 +1158,487 @@ def do_vcs_install(versionfile_source, ipy):
         f = open(".gitattributes", "a+")
         f.write("%s export-subst\n" % versionfile_source)
         f.close()
-        run_command([GIT, "add", ".gitattributes"])
+        files.append(".gitattributes")
+    run_command(GITS, ["add", "--"] + files)
+
 
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with "
+                  "prefix '%s'" % (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
 
 SHORT_VERSION_PY = """
-# This file was generated by 'versioneer.py' (0.7+) from
+# This file was generated by 'versioneer.py' (0.16) from
 # revision-control system data, or from the parent directory name of an
 # unpacked source archive. Distribution tarballs contain a pre-generated copy
 # of this file.
 
-version_version = '%(version)s'
-version_full = '%(full)s'
-def get_versions(default={}, verbose=False):
-    return {'version': version_version, 'full': version_full}
+import json
+import sys
+
+version_json = '''
+%s
+'''  # END VERSION_JSON
 
+
+def get_versions():
+    return json.loads(version_json)
 """
 
-DEFAULT = {"version": "unknown", "full": "unknown"}
 
 def versions_from_file(filename):
-    versions = {}
+    """Try to determine the version from _version.py if present."""
     try:
-        f = open(filename)
+        with open(filename) as f:
+            contents = f.read()
     except EnvironmentError:
-        return versions
-    for line in f.readlines():
-        mo = re.match("version_version = '([^']+)'", line)
-        if mo:
-            versions["version"] = mo.group(1)
-        mo = re.match("version_full = '([^']+)'", line)
-        if mo:
-            versions["full"] = mo.group(1)
-    f.close()
-    return versions
+        raise NotThisMethod("unable to read _version.py")
+    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
+                   contents, re.M | re.S)
+    if not mo:
+        raise NotThisMethod("no version_json in _version.py")
+    return json.loads(mo.group(1))
+
 
 def write_to_version_file(filename, versions):
-    f = open(filename, "w")
-    f.write(SHORT_VERSION_PY % versions)
-    f.close()
+    """Write the given version number to the given _version.py file."""
+    os.unlink(filename)
+    contents = json.dumps(versions, sort_keys=True,
+                          indent=1, separators=(",", ": "))
+    with open(filename, "w") as f:
+        f.write(SHORT_VERSION_PY % contents)
+
     print("set %s to '%s'" % (filename, versions["version"]))
 
 
-def get_best_versions(versionfile, tag_prefix, parentdir_prefix,
-                      default=DEFAULT, verbose=False):
-    # returns dict with two keys: 'version' and 'full'
-    #
-    # extract version from first of _version.py, 'git describe', parentdir.
-    # This is meant to work for developers using a source checkout, for users
-    # of a tarball created by 'setup.py sdist', and for users of a
-    # tarball/zipball created by 'git archive' or github's download-from-tag
-    # feature.
-
-    variables = get_expanded_variables(versionfile_source)
-    if variables:
-        ver = versions_from_expanded_variables(variables, tag_prefix)
-        if ver:
-            if verbose: print("got version from expanded variable %s" % ver)
-            return ver
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
 
-    ver = versions_from_file(versionfile)
-    if ver:
-        if verbose: print("got version from file %s %s" % (versionfile, ver))
-        return ver
 
-    ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if ver:
-        if verbose: print("got version from git %s" % ver)
-        return ver
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
 
-    ver = versions_from_parentdir(parentdir_prefix, versionfile_source, verbose)
-    if ver:
-        if verbose: print("got version from parentdir %s" % ver)
-        return ver
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
 
-    if verbose: print("got version from default %s" % ver)
-    return default
-
-def get_versions(default=DEFAULT, verbose=False):
-    assert versionfile_source is not None, "please set versioneer.versionfile_source"
-    assert tag_prefix is not None, "please set versioneer.tag_prefix"
-    assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix"
-    return get_best_versions(versionfile_source, tag_prefix, parentdir_prefix,
-                             default=default, verbose=verbose)
-def get_version(verbose=False):
-    return get_versions(verbose=verbose)["version"]
-
-class cmd_version(Command):
-    description = "report generated version string"
-    user_options = []
-    boolean_options = []
-    def initialize_options(self):
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+class VersioneerBadRootError(Exception):
+    """The project root directory is unknown or missing key files."""
+
+
+def get_versions(verbose=False):
+    """Get the project version from whatever source is available.
+
+    Returns dict with two keys: 'version' and 'full'.
+    """
+    if "versioneer" in sys.modules:
+        # see the discussion in cmdclass.py:get_cmdclass()
+        del sys.modules["versioneer"]
+
+    root = get_root()
+    cfg = get_config_from_root(root)
+
+    assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg"
+    handlers = HANDLERS.get(cfg.VCS)
+    assert handlers, "unrecognized VCS '%s'" % cfg.VCS
+    verbose = verbose or cfg.verbose
+    assert cfg.versionfile_source is not None, \
+        "please set versioneer.versionfile_source"
+    assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix"
+
+    versionfile_abs = os.path.join(root, cfg.versionfile_source)
+
+    # extract version from first of: _version.py, VCS command (e.g. 'git
+    # describe'), parentdir. This is meant to work for developers using a
+    # source checkout, for users of a tarball created by 'setup.py sdist',
+    # and for users of a tarball/zipball created by 'git archive' or github's
+    # download-from-tag feature or the equivalent in other VCSes.
+
+    get_keywords_f = handlers.get("get_keywords")
+    from_keywords_f = handlers.get("keywords")
+    if get_keywords_f and from_keywords_f:
+        try:
+            keywords = get_keywords_f(versionfile_abs)
+            ver = from_keywords_f(keywords, cfg.tag_prefix, verbose)
+            if verbose:
+                print("got version from expanded keyword %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        ver = versions_from_file(versionfile_abs)
+        if verbose:
+            print("got version from file %s %s" % (versionfile_abs, ver))
+        return ver
+    except NotThisMethod:
         pass
-    def finalize_options(self):
+
+    from_vcs_f = handlers.get("pieces_from_vcs")
+    if from_vcs_f:
+        try:
+            pieces = from_vcs_f(cfg.tag_prefix, root, verbose)
+            ver = render(pieces, cfg.style)
+            if verbose:
+                print("got version from VCS %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        if cfg.parentdir_prefix:
+            ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+            if verbose:
+                print("got version from parentdir %s" % ver)
+            return ver
+    except NotThisMethod:
         pass
-    def run(self):
-        ver = get_version(verbose=True)
-        print("Version is currently: %s" % ver)
-
-
-class cmd_build(_build):
-    def run(self):
-        versions = get_versions(verbose=True)
-        _build.run(self)
-        # now locate _version.py in the new build/ directory and replace it
-        # with an updated value
-        target_versionfile = os.path.join(self.build_lib, versionfile_build)
-        print("UPDATING %s" % target_versionfile)
-        os.unlink(target_versionfile)
-        f = open(target_versionfile, "w")
-        f.write(SHORT_VERSION_PY % versions)
-        f.close()
 
-class cmd_sdist(_sdist):
-    def run(self):
-        versions = get_versions(verbose=True)
-        self._versioneer_generated_versions = versions
-        # unless we update this, the command will keep using the old version
-        self.distribution.metadata.version = versions["version"]
-        return _sdist.run(self)
-
-    def make_release_tree(self, base_dir, files):
-        _sdist.make_release_tree(self, base_dir, files)
-        # now locate _version.py in the new base_dir directory (remembering
-        # that it may be a hardlink) and replace it with an updated value
-        target_versionfile = os.path.join(base_dir, versionfile_source)
-        print("UPDATING %s" % target_versionfile)
-        os.unlink(target_versionfile)
-        f = open(target_versionfile, "w")
-        f.write(SHORT_VERSION_PY % self._versioneer_generated_versions)
-        f.close()
+    if verbose:
+        print("unable to compute version")
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None, "error": "unable to compute version"}
+
+
+def get_version():
+    """Get the short version string for this project."""
+    return get_versions()["version"]
+
+
+def get_cmdclass():
+    """Get the custom setuptools/distutils subclasses used by Versioneer."""
+    if "versioneer" in sys.modules:
+        del sys.modules["versioneer"]
+        # this fixes the "python setup.py develop" case (also 'install' and
+        # 'easy_install .'), in which subdependencies of the main project are
+        # built (using setup.py bdist_egg) in the same python process. Assume
+        # a main project A and a dependency B, which use different versions
+        # of Versioneer. A's setup.py imports A's Versioneer, leaving it in
+        # sys.modules by the time B's setup.py is executed, causing B to run
+        # with the wrong versioneer. Setuptools wraps the sub-dep builds in a
+        # sandbox that restores sys.modules to it's pre-build state, so the
+        # parent is protected against the child's "import versioneer". By
+        # removing ourselves from sys.modules here, before the child build
+        # happens, we protect the child from the parent's versioneer too.
+        # Also see https://github.com/warner/python-versioneer/issues/52
+
+    cmds = {}
+
+    # we add "version" to both distutils and setuptools
+    from distutils.core import Command
+
+    class cmd_version(Command):
+        description = "report generated version string"
+        user_options = []
+        boolean_options = []
+
+        def initialize_options(self):
+            pass
+
+        def finalize_options(self):
+            pass
+
+        def run(self):
+            vers = get_versions(verbose=True)
+            print("Version: %s" % vers["version"])
+            print(" full-revisionid: %s" % vers.get("full-revisionid"))
+            print(" dirty: %s" % vers.get("dirty"))
+            if vers["error"]:
+                print(" error: %s" % vers["error"])
+    cmds["version"] = cmd_version
+
+    # we override "build_py" in both distutils and setuptools
+    #
+    # most invocation pathways end up running build_py:
+    #  distutils/build -> build_py
+    #  distutils/install -> distutils/build ->..
+    #  setuptools/bdist_wheel -> distutils/install ->..
+    #  setuptools/bdist_egg -> distutils/install_lib -> build_py
+    #  setuptools/install -> bdist_egg ->..
+    #  setuptools/develop -> ?
+
+    # we override different "build_py" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.build_py import build_py as _build_py
+    else:
+        from distutils.command.build_py import build_py as _build_py
+
+    class cmd_build_py(_build_py):
+        def run(self):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            versions = get_versions()
+            _build_py.run(self)
+            # now locate _version.py in the new build/ directory and replace
+            # it with an updated value
+            if cfg.versionfile_build:
+                target_versionfile = os.path.join(self.build_lib,
+                                                  cfg.versionfile_build)
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+    cmds["build_py"] = cmd_build_py
+
+    if "cx_Freeze" in sys.modules:  # cx_freeze enabled?
+        from cx_Freeze.dist import build_exe as _build_exe
+
+        class cmd_build_exe(_build_exe):
+            def run(self):
+                root = get_root()
+                cfg = get_config_from_root(root)
+                versions = get_versions()
+                target_versionfile = cfg.versionfile_source
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+
+                _build_exe.run(self)
+                os.unlink(target_versionfile)
+                with open(cfg.versionfile_source, "w") as f:
+                    LONG = LONG_VERSION_PY[cfg.VCS]
+                    f.write(LONG %
+                            {"DOLLAR": "$",
+                             "STYLE": cfg.style,
+                             "TAG_PREFIX": cfg.tag_prefix,
+                             "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                             "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                             })
+        cmds["build_exe"] = cmd_build_exe
+        del cmds["build_py"]
+
+    # we override different "sdist" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.sdist import sdist as _sdist
+    else:
+        from distutils.command.sdist import sdist as _sdist
+
+    class cmd_sdist(_sdist):
+        def run(self):
+            versions = get_versions()
+            self._versioneer_generated_versions = versions
+            # unless we update this, the command will keep using the old
+            # version
+            self.distribution.metadata.version = versions["version"]
+            return _sdist.run(self)
+
+        def make_release_tree(self, base_dir, files):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            _sdist.make_release_tree(self, base_dir, files)
+            # now locate _version.py in the new base_dir directory
+            # (remembering that it may be a hardlink) and replace it with an
+            # updated value
+            target_versionfile = os.path.join(base_dir, cfg.versionfile_source)
+            print("UPDATING %s" % target_versionfile)
+            write_to_version_file(target_versionfile,
+                                  self._versioneer_generated_versions)
+    cmds["sdist"] = cmd_sdist
+
+    return cmds
+
+
+CONFIG_ERROR = """
+setup.cfg is missing the necessary Versioneer configuration. You need
+a section like:
+
+ [versioneer]
+ VCS = git
+ style = pep440
+ versionfile_source = src/myproject/_version.py
+ versionfile_build = myproject/_version.py
+ tag_prefix =
+ parentdir_prefix = myproject-
+
+You will also need to edit your setup.py to use the results:
+
+ import versioneer
+ setup(version=versioneer.get_version(),
+       cmdclass=versioneer.get_cmdclass(), ...)
+
+Please read the docstring in ./versioneer.py for configuration instructions,
+edit setup.cfg, and re-run the installer or 'python versioneer.py setup'.
+"""
+
+SAMPLE_CONFIG = """
+# See the docstring in versioneer.py for instructions. Note that you must
+# re-run 'versioneer.py setup' after changing this section, and commit the
+# resulting files.
+
+[versioneer]
+#VCS = git
+#style = pep440
+#versionfile_source =
+#versionfile_build =
+#tag_prefix =
+#parentdir_prefix =
+
+"""
 
 INIT_PY_SNIPPET = """
 from ._version import get_versions
@@ -630,40 +1646,129 @@ __version__ = get_versions()['version']
 del get_versions
 """
 
-class cmd_update_files(Command):
-    description = "modify __init__.py and create _version.py"
-    user_options = []
-    boolean_options = []
-    def initialize_options(self):
-        pass
-    def finalize_options(self):
-        pass
-    def run(self):
-        ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py")
-        print(" creating %s" % versionfile_source)
-        f = open(versionfile_source, "w")
-        f.write(LONG_VERSION_PY % {"DOLLAR": "$",
-                                   "TAG_PREFIX": tag_prefix,
-                                   "PARENTDIR_PREFIX": parentdir_prefix,
-                                   "VERSIONFILE_SOURCE": versionfile_source,
-                                   })
-        f.close()
+
+def do_setup():
+    """Main VCS-independent setup function for installing Versioneer."""
+    root = get_root()
+    try:
+        cfg = get_config_from_root(root)
+    except (EnvironmentError, configparser.NoSectionError,
+            configparser.NoOptionError) as e:
+        if isinstance(e, (EnvironmentError, configparser.NoSectionError)):
+            print("Adding sample versioneer config to setup.cfg",
+                  file=sys.stderr)
+            with open(os.path.join(root, "setup.cfg"), "a") as f:
+                f.write(SAMPLE_CONFIG)
+        print(CONFIG_ERROR, file=sys.stderr)
+        return 1
+
+    print(" creating %s" % cfg.versionfile_source)
+    with open(cfg.versionfile_source, "w") as f:
+        LONG = LONG_VERSION_PY[cfg.VCS]
+        f.write(LONG % {"DOLLAR": "$",
+                        "STYLE": cfg.style,
+                        "TAG_PREFIX": cfg.tag_prefix,
+                        "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                        "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                        })
+
+    ipy = os.path.join(os.path.dirname(cfg.versionfile_source),
+                       "__init__.py")
+    if os.path.exists(ipy):
         try:
-            old = open(ipy, "r").read()
+            with open(ipy, "r") as f:
+                old = f.read()
         except EnvironmentError:
             old = ""
         if INIT_PY_SNIPPET not in old:
             print(" appending to %s" % ipy)
-            f = open(ipy, "a")
-            f.write(INIT_PY_SNIPPET)
-            f.close()
+            with open(ipy, "a") as f:
+                f.write(INIT_PY_SNIPPET)
         else:
             print(" %s unmodified" % ipy)
-        do_vcs_install(versionfile_source, ipy)
+    else:
+        print(" %s doesn't exist, ok" % ipy)
+        ipy = None
+
+    # Make sure both the top-level "versioneer.py" and versionfile_source
+    # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so
+    # they'll be copied into source distributions. Pip won't be able to
+    # install the package without this.
+    manifest_in = os.path.join(root, "MANIFEST.in")
+    simple_includes = set()
+    try:
+        with open(manifest_in, "r") as f:
+            for line in f:
+                if line.startswith("include "):
+                    for include in line.split()[1:]:
+                        simple_includes.add(include)
+    except EnvironmentError:
+        pass
+    # That doesn't cover everything MANIFEST.in can do
+    # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
+    # it might give some false negatives. Appending redundant 'include'
+    # lines is safe, though.
+    if "versioneer.py" not in simple_includes:
+        print(" appending 'versioneer.py' to MANIFEST.in")
+        with open(manifest_in, "a") as f:
+            f.write("include versioneer.py\n")
+    else:
+        print(" 'versioneer.py' already in MANIFEST.in")
+    if cfg.versionfile_source not in simple_includes:
+        print(" appending versionfile_source ('%s') to MANIFEST.in" %
+              cfg.versionfile_source)
+        with open(manifest_in, "a") as f:
+            f.write("include %s\n" % cfg.versionfile_source)
+    else:
+        print(" versionfile_source already in MANIFEST.in")
 
-def get_cmdclass():
-    return {'version': cmd_version,
-            'update_files': cmd_update_files,
-            'build': cmd_build,
-            'sdist': cmd_sdist,
-            }
+    # Make VCS-specific changes. For git, this means creating/changing
+    # .gitattributes to mark _version.py for export-time keyword
+    # substitution.
+    do_vcs_install(manifest_in, cfg.versionfile_source, ipy)
+    return 0
+
+
+def scan_setup_py():
+    """Validate the contents of setup.py against Versioneer's expectations."""
+    found = set()
+    setters = False
+    errors = 0
+    with open("setup.py", "r") as f:
+        for line in f.readlines():
+            if "import versioneer" in line:
+                found.add("import")
+            if "versioneer.get_cmdclass()" in line:
+                found.add("cmdclass")
+            if "versioneer.get_version()" in line:
+                found.add("get_version")
+            if "versioneer.VCS" in line:
+                setters = True
+            if "versioneer.versionfile_source" in line:
+                setters = True
+    if len(found) != 3:
+        print("")
+        print("Your setup.py appears to be missing some important items")
+        print("(but I might be wrong). Please make sure it has something")
+        print("roughly like the following:")
+        print("")
+        print(" import versioneer")
+        print(" setup( version=versioneer.get_version(),")
+        print("        cmdclass=versioneer.get_cmdclass(),  ...)")
+        print("")
+        errors += 1
+    if setters:
+        print("You should remove lines like 'versioneer.VCS = ' and")
+        print("'versioneer.versionfile_source = ' . This configuration")
+        print("now lives in setup.cfg, and should be removed from setup.py")
+        print("")
+        errors += 1
+    return errors
+
+if __name__ == "__main__":
+    cmd = sys.argv[1]
+    if cmd == "setup":
+        errors = do_setup()
+        errors += scan_setup_py()
+        if errors:
+            sys.exit(1)
-- 
cgit v1.2.3


From 86d89807b1ce0eeaaf422143ad9805237c6cb407 Mon Sep 17 00:00:00 2001
From: Bruno Wagner <bwagner@riseup.net>
Date: Mon, 11 Apr 2016 11:57:58 -0300
Subject: [style] Removed duplicated import

There was a duplicate import for get_versions, that was not at
the top of the file, that caused a pep warning and was fixed
in this commit
---
 src/leap/keymanager/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 91cd2f8..ce49667 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -922,6 +922,5 @@ def _get_domain(url):
     """
     return urlparse(url).hostname
 
-from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions
-- 
cgit v1.2.3


From f719b8546a214b6bd6ecbef715c5f34b42ad8d52 Mon Sep 17 00:00:00 2001
From: "Kali Kaneko (leap communications)" <kali@leap.se>
Date: Mon, 11 Apr 2016 12:14:01 -0400
Subject: [bug] delete versioneer duplicated block

---
 src/leap/keymanager/__init__.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index ce49667..1106c23 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -921,6 +921,3 @@ def _get_domain(url):
     :rtype: str
     """
     return urlparse(url).hostname
-
-__version__ = get_versions()['version']
-del get_versions
-- 
cgit v1.2.3


From fe46507c44fbfb9c8ed72799d5e8bfb0ac32a10c Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Fri, 8 Apr 2016 18:22:14 +0200
Subject: [feat] reduce log level for encrypt/decrypt errors

* Related: #8022
---
 src/leap/keymanager/openpgp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index 0d5a866..a843261 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -680,7 +680,7 @@ class OpenPGPScheme(EncryptionScheme):
                 self._assert_gpg_result_ok(result)
                 defer.returnValue(result.data)
             except errors.GPGError as e:
-                logger.error('Failed to decrypt: %s.' % str(e))
+                logger.warning('Failed to encrypt: %s.' % str(e))
                 raise errors.EncryptError()
 
     @defer.inlineCallbacks
@@ -726,7 +726,7 @@ class OpenPGPScheme(EncryptionScheme):
 
                 defer.returnValue((result.data, sign_valid))
             except errors.GPGError as e:
-                logger.error('Failed to decrypt: %s.' % str(e))
+                logger.warning('Failed to decrypt: %s.' % str(e))
                 raise errors.DecryptError(str(e))
 
     def is_encrypted(self, data):
-- 
cgit v1.2.3


From 90a1df64f6a5df09a289eeb322a77e2e984569aa Mon Sep 17 00:00:00 2001
From: "Kali Kaneko (leap communications)" <kali@leap.se>
Date: Mon, 18 Apr 2016 11:04:54 -0400
Subject: [pkg] Update changelog

---
 CHANGELOG                  | 95 ----------------------------------------------
 CHANGELOG.rst              | 19 ++++++++++
 HISTORY.rst                | 95 ++++++++++++++++++++++++++++++++++++++++++++++
 changes/next-changelog.txt | 12 +-----
 4 files changed, 115 insertions(+), 106 deletions(-)
 delete mode 100644 CHANGELOG
 create mode 100644 CHANGELOG.rst
 create mode 100644 HISTORY.rst

diff --git a/CHANGELOG b/CHANGELOG
deleted file mode 100644
index 0a9221d..0000000
--- a/CHANGELOG
+++ /dev/null
@@ -1,95 +0,0 @@
-0.4.3 Oct 28, 2015:
-  o self-repair the keyring if keys get duplicated. Closes: #7498
-  o catch request exceptions on key fetching. Closes #7410.
-  o Don't repush a public key with different addres
-  o use async events api. Closes #7224.
-  o Use ca_bundle when fetching keys by url.
-  o add logging to fetch_key. Related: #7410.
-  o more verbosity in get_key wrong address log.
-  o don't repush a public key with different address. Related #7420.
-
-0.4.2 Aug 26, 2015:
-  o Style changes.
-  o Tests updates.
-  o Packaging improvements.
-
-0.4.1 Jul 10, 2015:
-  o Remove the dependency on enum34. Closes: #7188.
-
-0.4.0 Jun 8, 2015:
-  o Adapt to new events api on leap.common. Related to #5359.
-  o Add 'fetch_key' method to fetch keys from a URI. Closes #5932.
-  o Clean up API.
-  o Fix call to python-gnupg's verify_file() method. Closes #6022.
-  o KeyManager.put_key now accepts also ascii keys.
-  o Multi uid support. Closes #6212.
-  o Port keymanager to the new soledad async API. Closes #6368.
-  o Return always KeyNotFound failure if fetch keys fails on an unknown error.
-  o Upgrade keys if not successfully used and strict high validation level.
-    Closes #6211.
-  o Use addresses instead of keys for encrypt, decrypt, sign & verify.
-    Closes #6346.
-  o Expose info about the signing key. Closes #6366.
-  o Fetched keys from other domain than its provider are set as 'Weak Chain'
-    validation level. Closes #6815.
-  o Keep old key after upgrade. Closes #6262.
-  o New soledad doc struct for encryption-keys. Closes #6299.
-  o Upgrade key when signed by old key. Closes #6240.
-
-0.3.8 Apr 4, 2014:
-  o Properly raise KeyNotFound exception when looking for keys on
-    nickserver. Fixes #5415.
-  o Do not decode decrypted data, return as str.
-  o Use a better version handler for the gnupg version check.
-  o Memoize call to get_key. Closes #4784.
-  o Update auth to interact with webapp v2. Fixes #5120.
-
-0.3.7 Dec 6, 2013:
-  o Fix error return values on openpgp backend.
-  o Remove address check when sending email and rely in the email
-    client to verify that is correct. Closes #4491.
-  o Support sending encrypted mails to addresses using the '+' sign.
-  o Improve exception names and handling.
-
-0.3.6 Nov 15, 2013:
-  o Default encoding to 'utf-8' in case of system encodings not
-    set. Closes #4427.
-  o Add verification of detached signatures. Closes #4375.
-  o New openpgp method: parse_ascii_keys.
-  o Expose openpgp methods in keymanager (parse_ascii_keys, put_key,
-    delete_key).
-
-0.3.5 Nov 1, 2013:
-  o Return unicode decrypted text to avoid encoding issues. Related to
-    #4000.
-
-0.3.4 Oct 18, 2013:
-  o Add option to choose cipher and digest algorithms when signing and
-    encrypting. Closes #4030.
-
-0.3.3 Oct 4, 2013:
-  o Add a sanity check for the correct version of gnupg.
-  o Update code to use gnupg 1.2.2 python module. Closes #2342.
-
-0.3.2 Sep 6, 2013:
-  o Do not raise exception when a GET request doesn't return 2XX
-    code. Nickserver uses error codes for more verbosity in the
-    result.
-  o Accept unicode ascii keys along with str.
-
-0.3.1 Aug 23, 2013:
-  o Signal different key related events, like key generation, key
-    upload.
-  o Update to new soledad package scheme (common, client and
-    server). Closes #3487.
-  o Packaging improvements: add versioneer and parse_requirements.
-
-0.3.0 Aug 9, 2013:
-  o If a nickserver request fails in any way, notify and continue.
-  o Options parameter in gnupg.GPG isn't supported by all versions, so
-    removing it for the time being.
-  o Add support for bundled gpg. Closes #3397.
-  o Refactor API to include encrypt/decrypt/sign/verify in KeyManager.
-
-0.2.0 Jul 12, 2013:
-  o Move keymanager to its own package
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
new file mode 100644
index 0000000..f1cd9e1
--- /dev/null
+++ b/CHANGELOG.rst
@@ -0,0 +1,19 @@
+0.5.0 - 18 Apr, 2016
+++++++++++++++++++++
+
+Features
+~~~~~~~~
+- `#7485 <https://leap.se/code/issues/7485>`_: Move validation, usage and audited date to the active document.
+- `#7713 <https://leap.se/code/issues/7713>`_: Update soledad documents by adding versioning field.
+- `#7500 <https://leap.se/code/issues/7500>`_: Use fingerprints instead of key ids.
+- `#7712 <https://leap.se/code/issues/7712>`_: Document the soledad docs fields.
+- Make EncryptionKey aware of the active address.
+
+Bugfixes
+~~~~~~~~
+- `#7974 <https://leap.se/code/issues/7974>`_: Return KeyNotFound Failure if not valid key is given to put_raw_key.
+
+Misc
+~~~~
+- This version includes changes in the Soledad Documents and minor modifications to the API.
+- Changelog migrated to rst.
diff --git a/HISTORY.rst b/HISTORY.rst
new file mode 100644
index 0000000..0a9221d
--- /dev/null
+++ b/HISTORY.rst
@@ -0,0 +1,95 @@
+0.4.3 Oct 28, 2015:
+  o self-repair the keyring if keys get duplicated. Closes: #7498
+  o catch request exceptions on key fetching. Closes #7410.
+  o Don't repush a public key with different addres
+  o use async events api. Closes #7224.
+  o Use ca_bundle when fetching keys by url.
+  o add logging to fetch_key. Related: #7410.
+  o more verbosity in get_key wrong address log.
+  o don't repush a public key with different address. Related #7420.
+
+0.4.2 Aug 26, 2015:
+  o Style changes.
+  o Tests updates.
+  o Packaging improvements.
+
+0.4.1 Jul 10, 2015:
+  o Remove the dependency on enum34. Closes: #7188.
+
+0.4.0 Jun 8, 2015:
+  o Adapt to new events api on leap.common. Related to #5359.
+  o Add 'fetch_key' method to fetch keys from a URI. Closes #5932.
+  o Clean up API.
+  o Fix call to python-gnupg's verify_file() method. Closes #6022.
+  o KeyManager.put_key now accepts also ascii keys.
+  o Multi uid support. Closes #6212.
+  o Port keymanager to the new soledad async API. Closes #6368.
+  o Return always KeyNotFound failure if fetch keys fails on an unknown error.
+  o Upgrade keys if not successfully used and strict high validation level.
+    Closes #6211.
+  o Use addresses instead of keys for encrypt, decrypt, sign & verify.
+    Closes #6346.
+  o Expose info about the signing key. Closes #6366.
+  o Fetched keys from other domain than its provider are set as 'Weak Chain'
+    validation level. Closes #6815.
+  o Keep old key after upgrade. Closes #6262.
+  o New soledad doc struct for encryption-keys. Closes #6299.
+  o Upgrade key when signed by old key. Closes #6240.
+
+0.3.8 Apr 4, 2014:
+  o Properly raise KeyNotFound exception when looking for keys on
+    nickserver. Fixes #5415.
+  o Do not decode decrypted data, return as str.
+  o Use a better version handler for the gnupg version check.
+  o Memoize call to get_key. Closes #4784.
+  o Update auth to interact with webapp v2. Fixes #5120.
+
+0.3.7 Dec 6, 2013:
+  o Fix error return values on openpgp backend.
+  o Remove address check when sending email and rely in the email
+    client to verify that is correct. Closes #4491.
+  o Support sending encrypted mails to addresses using the '+' sign.
+  o Improve exception names and handling.
+
+0.3.6 Nov 15, 2013:
+  o Default encoding to 'utf-8' in case of system encodings not
+    set. Closes #4427.
+  o Add verification of detached signatures. Closes #4375.
+  o New openpgp method: parse_ascii_keys.
+  o Expose openpgp methods in keymanager (parse_ascii_keys, put_key,
+    delete_key).
+
+0.3.5 Nov 1, 2013:
+  o Return unicode decrypted text to avoid encoding issues. Related to
+    #4000.
+
+0.3.4 Oct 18, 2013:
+  o Add option to choose cipher and digest algorithms when signing and
+    encrypting. Closes #4030.
+
+0.3.3 Oct 4, 2013:
+  o Add a sanity check for the correct version of gnupg.
+  o Update code to use gnupg 1.2.2 python module. Closes #2342.
+
+0.3.2 Sep 6, 2013:
+  o Do not raise exception when a GET request doesn't return 2XX
+    code. Nickserver uses error codes for more verbosity in the
+    result.
+  o Accept unicode ascii keys along with str.
+
+0.3.1 Aug 23, 2013:
+  o Signal different key related events, like key generation, key
+    upload.
+  o Update to new soledad package scheme (common, client and
+    server). Closes #3487.
+  o Packaging improvements: add versioneer and parse_requirements.
+
+0.3.0 Aug 9, 2013:
+  o If a nickserver request fails in any way, notify and continue.
+  o Options parameter in gnupg.GPG isn't supported by all versions, so
+    removing it for the time being.
+  o Add support for bundled gpg. Closes #3397.
+  o Refactor API to include encrypt/decrypt/sign/verify in KeyManager.
+
+0.2.0 Jul 12, 2013:
+  o Move keymanager to its own package
diff --git a/changes/next-changelog.txt b/changes/next-changelog.txt
index 2dd34d4..c59c85a 100644
--- a/changes/next-changelog.txt
+++ b/changes/next-changelog.txt
@@ -1,4 +1,4 @@
-0.5.0 - xxx
+0.5.1 - xxx
 +++++++++++++++++++++++++++++++
 
 Please add lines to this file, they will be moved to the CHANGELOG.rst during
@@ -10,26 +10,16 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff.
 
 Features
 ~~~~~~~~
-- `#7485 <https://leap.se/code/issues/7485>`_: Move validation, usage and audited date to the active document.
-- `#7713 <https://leap.se/code/issues/7713>`_: Update soledad documents by adding versioning field.
-- `#7500 <https://leap.se/code/issues/7500>`_: Use fingerprints instead of key ids.
-- `#7712 <https://leap.se/code/issues/7712>`_: Document the soledad docs fields.
-- Make EncryptionKey aware of the active address.
-
 - `#1234 <https://leap.se/code/issues/1234>`_: Description of the new feature corresponding with issue #1234.
 - New feature without related issue number.
 
 Bugfixes
 ~~~~~~~~
-- `#7974 <https://leap.se/code/issues/7974>`_: Return KeyNotFound Failure if not valid key is given to put_raw_key.
-
 - `#1235 <https://leap.se/code/issues/1235>`_: Description for the fixed stuff corresponding with issue #1235.
 - Bugfix without related issue number.
 
 Misc
 ~~~~
-- This version includes changes in the Soledad Documents and minor modifications to the API.
-
 - `#1236 <https://leap.se/code/issues/1236>`_: Description of the new feature corresponding with issue #1236.
 - Some change without issue number.
 
-- 
cgit v1.2.3


From 4a532500e3a6130199a10622d17f10a247609f06 Mon Sep 17 00:00:00 2001
From: elijah <elijah@riseup.net>
Date: Mon, 18 Apr 2016 23:48:17 -0700
Subject: [bug] encryption keys can now be serialized to json using
 key.get_dict()

---
 changes/bug-serializable-keys |  1 +
 src/leap/keymanager/keys.py   | 14 ++++++++++++++
 2 files changed, 15 insertions(+)
 create mode 100644 changes/bug-serializable-keys

diff --git a/changes/bug-serializable-keys b/changes/bug-serializable-keys
new file mode 100644
index 0000000..0885fd2
--- /dev/null
+++ b/changes/bug-serializable-keys
@@ -0,0 +1 @@
+- encryption keys can now be serialized to json using key.get_dict()
diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index 38d66b5..34d921c 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -258,6 +258,20 @@ class EncryptionKey(object):
             KEY_TAGS_KEY: [KEYMANAGER_ACTIVE_TAG],
         })
 
+    def get_dict(self):
+        """
+        :return: a serializable dict representation of this key.
+        :rtype: dict
+        """
+        return {
+          KEY_ADDRESS_KEY: self.address,
+          KEY_UIDS_KEY: self.uids,
+          KEY_FINGERPRINT_KEY: self.fingerprint,
+          KEY_LENGTH_KEY: self.length,
+          KEY_EXPIRY_DATE_KEY: str(self.expiry_date),
+          KEY_DATA_KEY: self.key_data
+        }
+
     def __repr__(self):
         """
         Representation of this class
-- 
cgit v1.2.3


From 033049c64ebad6edd2ab1a9a1b359c53cbfb46e6 Mon Sep 17 00:00:00 2001
From: "Kali Kaneko (leap communications)" <kali@leap.se>
Date: Mon, 25 Apr 2016 19:18:46 -0400
Subject: freeze debian version

---
 src/leap/keymanager/_version.py | 490 +---------------------------------------
 1 file changed, 10 insertions(+), 480 deletions(-)

diff --git a/src/leap/keymanager/_version.py b/src/leap/keymanager/_version.py
index b28c697..8e7e599 100644
--- a/src/leap/keymanager/_version.py
+++ b/src/leap/keymanager/_version.py
@@ -1,484 +1,14 @@
 
-# This file helps to compute a version number in source trees obtained from
-# git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (built by setup.py sdist) and build
-# directories (produced by setup.py build) will contain a much shorter file
-# that just contains the computed version number.
+# This file was generated by the `freeze_debianver` command in setup.py
+# Using 'versioneer.py' (0.16) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
 
-# This file is released into the public domain. Generated by
-# versioneer-0.16 (https://github.com/warner/python-versioneer)
+version_version = '0.5.0'
+full_revisionid = '0f600595dd2d26bc16174435dbf0c3accd690982'
 
-"""Git implementation of _version.py."""
 
-import errno
-import os
-import re
-import subprocess
-import sys
-
-
-def get_keywords():
-    """Get the keywords needed to look up the version information."""
-    # these strings will be replaced by git during git-archive.
-    # setup.py/versioneer.py will grep for the variable names, so they must
-    # each be defined on a line of their own. _version.py will just call
-    # get_keywords().
-    git_refnames = "$Format:%d$"
-    git_full = "$Format:%H$"
-    keywords = {"refnames": git_refnames, "full": git_full}
-    return keywords
-
-
-class VersioneerConfig:
-    """Container for Versioneer configuration parameters."""
-
-
-def get_config():
-    """Create, populate and return the VersioneerConfig() object."""
-    # these strings are filled in when 'setup.py versioneer' creates
-    # _version.py
-    cfg = VersioneerConfig()
-    cfg.VCS = "git"
-    cfg.style = "pep440"
-    cfg.tag_prefix = ""
-    cfg.parentdir_prefix = "None"
-    cfg.versionfile_source = "src/leap/keymanager/_version.py"
-    cfg.verbose = False
-    return cfg
-
-
-class NotThisMethod(Exception):
-    """Exception raised if a method is not valid for the current scenario."""
-
-
-LONG_VERSION_PY = {}
-HANDLERS = {}
-
-
-def register_vcs_handler(vcs, method):  # decorator
-    """Decorator to mark a method as the handler for a particular VCS."""
-    def decorate(f):
-        """Store f in HANDLERS[vcs][method]."""
-        if vcs not in HANDLERS:
-            HANDLERS[vcs] = {}
-        HANDLERS[vcs][method] = f
-        return f
-    return decorate
-
-
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
-    """Call the given command(s)."""
-    assert isinstance(commands, list)
-    p = None
-    for c in commands:
-        try:
-            dispcmd = str([c] + args)
-            # remember shell=False, so use git.cmd on windows, not just git
-            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
-                                 stderr=(subprocess.PIPE if hide_stderr
-                                         else None))
-            break
-        except EnvironmentError:
-            e = sys.exc_info()[1]
-            if e.errno == errno.ENOENT:
-                continue
-            if verbose:
-                print("unable to run %s" % dispcmd)
-                print(e)
-            return None
-    else:
-        if verbose:
-            print("unable to find command, tried %s" % (commands,))
-        return None
-    stdout = p.communicate()[0].strip()
-    if sys.version_info[0] >= 3:
-        stdout = stdout.decode()
-    if p.returncode != 0:
-        if verbose:
-            print("unable to run %s (error)" % dispcmd)
-        return None
-    return stdout
-
-
-def versions_from_parentdir(parentdir_prefix, root, verbose):
-    """Try to determine the version from the parent directory name.
-
-    Source tarballs conventionally unpack into a directory that includes
-    both the project name and a version string.
-    """
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start with "
-                  "prefix '%s'" % (root, dirname, parentdir_prefix))
-        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
-    return {"version": dirname[len(parentdir_prefix):],
-            "full-revisionid": None,
-            "dirty": False, "error": None}
-
-
-@register_vcs_handler("git", "get_keywords")
-def git_get_keywords(versionfile_abs):
-    """Extract version information from the given file."""
-    # the code embedded in _version.py can just fetch the value of these
-    # keywords. When used from setup.py, we don't want to import _version.py,
-    # so we do it with a regexp instead. This function is not used from
-    # _version.py.
-    keywords = {}
-    try:
-        f = open(versionfile_abs, "r")
-        for line in f.readlines():
-            if line.strip().startswith("git_refnames ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["refnames"] = mo.group(1)
-            if line.strip().startswith("git_full ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["full"] = mo.group(1)
-        f.close()
-    except EnvironmentError:
-        pass
-    return keywords
-
-
-@register_vcs_handler("git", "keywords")
-def git_versions_from_keywords(keywords, tag_prefix, verbose):
-    """Get version information from git keywords."""
-    if not keywords:
-        raise NotThisMethod("no keywords at all, weird")
-    refnames = keywords["refnames"].strip()
-    if refnames.startswith("$Format"):
-        if verbose:
-            print("keywords are unexpanded, not using")
-        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
-    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
-    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
-    TAG = "tag: "
-    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
-    if not tags:
-        # Either we're using git < 1.8.3, or there really are no tags. We use
-        # a heuristic: assume all version tags have a digit. The old git %d
-        # expansion behaves like git log --decorate=short and strips out the
-        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
-        # between branches and tags. By ignoring refnames without digits, we
-        # filter out many common branch names like "release" and
-        # "stabilization", as well as "HEAD" and "master".
-        tags = set([r for r in refs if re.search(r'\d', r)])
-        if verbose:
-            print("discarding '%s', no digits" % ",".join(refs-tags))
-    if verbose:
-        print("likely tags: %s" % ",".join(sorted(tags)))
-    for ref in sorted(tags):
-        # sorting will prefer e.g. "2.0" over "2.0rc1"
-        if ref.startswith(tag_prefix):
-            r = ref[len(tag_prefix):]
-            if verbose:
-                print("picking %s" % r)
-            return {"version": r,
-                    "full-revisionid": keywords["full"].strip(),
-                    "dirty": False, "error": None
-                    }
-    # no suitable tags, so version is "0+unknown", but full hex is still there
-    if verbose:
-        print("no suitable tags, using unknown + full revision id")
-    return {"version": "0+unknown",
-            "full-revisionid": keywords["full"].strip(),
-            "dirty": False, "error": "no suitable tags"}
-
-
-@register_vcs_handler("git", "pieces_from_vcs")
-def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
-    """Get version from 'git describe' in the root of the source tree.
-
-    This only gets called if the git-archive 'subst' keywords were *not*
-    expanded, and _version.py hasn't already been rewritten with a short
-    version string, meaning we're inside a checked out source tree.
-    """
-    if not os.path.exists(os.path.join(root, ".git")):
-        if verbose:
-            print("no .git in %s" % root)
-        raise NotThisMethod("no .git directory")
-
-    GITS = ["git"]
-    if sys.platform == "win32":
-        GITS = ["git.cmd", "git.exe"]
-    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
-    # if there isn't one, this yields HEX[-dirty] (no NUM)
-    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
-                                      "--always", "--long",
-                                      "--match", "%s*" % tag_prefix],
-                               cwd=root)
-    # --long was added in git-1.5.5
-    if describe_out is None:
-        raise NotThisMethod("'git describe' failed")
-    describe_out = describe_out.strip()
-    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
-    if full_out is None:
-        raise NotThisMethod("'git rev-parse' failed")
-    full_out = full_out.strip()
-
-    pieces = {}
-    pieces["long"] = full_out
-    pieces["short"] = full_out[:7]  # maybe improved later
-    pieces["error"] = None
-
-    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
-    # TAG might have hyphens.
-    git_describe = describe_out
-
-    # look for -dirty suffix
-    dirty = git_describe.endswith("-dirty")
-    pieces["dirty"] = dirty
-    if dirty:
-        git_describe = git_describe[:git_describe.rindex("-dirty")]
-
-    # now we have TAG-NUM-gHEX or HEX
-
-    if "-" in git_describe:
-        # TAG-NUM-gHEX
-        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
-        if not mo:
-            # unparseable. Maybe git-describe is misbehaving?
-            pieces["error"] = ("unable to parse git-describe output: '%s'"
-                               % describe_out)
-            return pieces
-
-        # tag
-        full_tag = mo.group(1)
-        if not full_tag.startswith(tag_prefix):
-            if verbose:
-                fmt = "tag '%s' doesn't start with prefix '%s'"
-                print(fmt % (full_tag, tag_prefix))
-            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
-                               % (full_tag, tag_prefix))
-            return pieces
-        pieces["closest-tag"] = full_tag[len(tag_prefix):]
-
-        # distance: number of commits since tag
-        pieces["distance"] = int(mo.group(2))
-
-        # commit: short hex revision ID
-        pieces["short"] = mo.group(3)
-
-    else:
-        # HEX: no tags
-        pieces["closest-tag"] = None
-        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
-                                cwd=root)
-        pieces["distance"] = int(count_out)  # total number of commits
-
-    return pieces
-
-
-def plus_or_dot(pieces):
-    """Return a + if we don't already have one, else return a ."""
-    if "+" in pieces.get("closest-tag", ""):
-        return "."
-    return "+"
-
-
-def render_pep440(pieces):
-    """Build up version string, with post-release "local version identifier".
-
-    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
-    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
-
-    Exceptions:
-    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += plus_or_dot(pieces)
-            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
-            if pieces["dirty"]:
-                rendered += ".dirty"
-    else:
-        # exception #1
-        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
-                                          pieces["short"])
-        if pieces["dirty"]:
-            rendered += ".dirty"
-    return rendered
-
-
-def render_pep440_pre(pieces):
-    """TAG[.post.devDISTANCE] -- No -dirty.
-
-    Exceptions:
-    1: no tags. 0.post.devDISTANCE
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"]:
-            rendered += ".post.dev%d" % pieces["distance"]
-    else:
-        # exception #1
-        rendered = "0.post.dev%d" % pieces["distance"]
-    return rendered
-
-
-def render_pep440_post(pieces):
-    """TAG[.postDISTANCE[.dev0]+gHEX] .
-
-    The ".dev0" means dirty. Note that .dev0 sorts backwards
-    (a dirty tree will appear "older" than the corresponding clean one),
-    but you shouldn't be releasing software with -dirty anyways.
-
-    Exceptions:
-    1: no tags. 0.postDISTANCE[.dev0]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += ".post%d" % pieces["distance"]
-            if pieces["dirty"]:
-                rendered += ".dev0"
-            rendered += plus_or_dot(pieces)
-            rendered += "g%s" % pieces["short"]
-    else:
-        # exception #1
-        rendered = "0.post%d" % pieces["distance"]
-        if pieces["dirty"]:
-            rendered += ".dev0"
-        rendered += "+g%s" % pieces["short"]
-    return rendered
-
-
-def render_pep440_old(pieces):
-    """TAG[.postDISTANCE[.dev0]] .
-
-    The ".dev0" means dirty.
-
-    Eexceptions:
-    1: no tags. 0.postDISTANCE[.dev0]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += ".post%d" % pieces["distance"]
-            if pieces["dirty"]:
-                rendered += ".dev0"
-    else:
-        # exception #1
-        rendered = "0.post%d" % pieces["distance"]
-        if pieces["dirty"]:
-            rendered += ".dev0"
-    return rendered
-
-
-def render_git_describe(pieces):
-    """TAG[-DISTANCE-gHEX][-dirty].
-
-    Like 'git describe --tags --dirty --always'.
-
-    Exceptions:
-    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"]:
-            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
-    else:
-        # exception #1
-        rendered = pieces["short"]
-    if pieces["dirty"]:
-        rendered += "-dirty"
-    return rendered
-
-
-def render_git_describe_long(pieces):
-    """TAG-DISTANCE-gHEX[-dirty].
-
-    Like 'git describe --tags --dirty --always -long'.
-    The distance/hash is unconditional.
-
-    Exceptions:
-    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
-    else:
-        # exception #1
-        rendered = pieces["short"]
-    if pieces["dirty"]:
-        rendered += "-dirty"
-    return rendered
-
-
-def render(pieces, style):
-    """Render the given version pieces into the requested style."""
-    if pieces["error"]:
-        return {"version": "unknown",
-                "full-revisionid": pieces.get("long"),
-                "dirty": None,
-                "error": pieces["error"]}
-
-    if not style or style == "default":
-        style = "pep440"  # the default
-
-    if style == "pep440":
-        rendered = render_pep440(pieces)
-    elif style == "pep440-pre":
-        rendered = render_pep440_pre(pieces)
-    elif style == "pep440-post":
-        rendered = render_pep440_post(pieces)
-    elif style == "pep440-old":
-        rendered = render_pep440_old(pieces)
-    elif style == "git-describe":
-        rendered = render_git_describe(pieces)
-    elif style == "git-describe-long":
-        rendered = render_git_describe_long(pieces)
-    else:
-        raise ValueError("unknown style '%s'" % style)
-
-    return {"version": rendered, "full-revisionid": pieces["long"],
-            "dirty": pieces["dirty"], "error": None}
-
-
-def get_versions():
-    """Get version information or return default if unable to do so."""
-    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
-    # __file__, we can work backwards from there to the root. Some
-    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
-    # case we can only use expanded keywords.
-
-    cfg = get_config()
-    verbose = cfg.verbose
-
-    try:
-        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
-                                          verbose)
-    except NotThisMethod:
-        pass
-
-    try:
-        root = os.path.realpath(__file__)
-        # versionfile_source is the relative path from the top of the source
-        # tree (where the .git directory might live) to this file. Invert
-        # this to find the root from __file__.
-        for i in cfg.versionfile_source.split('/'):
-            root = os.path.dirname(root)
-    except NameError:
-        return {"version": "0+unknown", "full-revisionid": None,
-                "dirty": None,
-                "error": "unable to find root of source tree"}
-
-    try:
-        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
-        return render(pieces, cfg.style)
-    except NotThisMethod:
-        pass
-
-    try:
-        if cfg.parentdir_prefix:
-            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
-    except NotThisMethod:
-        pass
-
-    return {"version": "0+unknown", "full-revisionid": None,
-            "dirty": None,
-            "error": "unable to compute version"}
+def get_versions(default={}, verbose=False):
+        return {'version': version_version,
+                'full-revisionid': full_revisionid}
-- 
cgit v1.2.3


From f47e125e74fe6fd3a68110b34de162e928e98df3 Mon Sep 17 00:00:00 2001
From: "Kali Kaneko (leap communications)" <kali@leap.se>
Date: Mon, 25 Apr 2016 19:19:20 -0400
Subject: update debian changelog to 0.5.0 release

---
 debian/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index 420db7f..de46a4d 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+leap-keymanager (0.5.0) unstable; urgency=medium
+
+  * Update to 0.5.0 release. 
+
+ -- Ben Carrillo <ben@futeisha.org>  Mon, 25 Apr 2016 19:18:49 -0400
+
 leap-keymanager (0.4.3) unstable; urgency=medium
 
   * Update to 0.4.3 release. 
-- 
cgit v1.2.3


From cd4f6648134d39e018ac5ac0c085f1dc57be0104 Mon Sep 17 00:00:00 2001
From: "Kali Kaneko (leap communications)" <kali@leap.se>
Date: Mon, 25 Apr 2016 19:20:07 -0400
Subject: update changelog path

---
 debian/rules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/debian/rules b/debian/rules
index 2bdcd17..40e9a09 100755
--- a/debian/rules
+++ b/debian/rules
@@ -4,6 +4,6 @@
 	dh $@ --with python2 --buildsystem=python_distutils
 
 override_dh_installchangelogs:
-	 dh_installchangelogs CHANGELOG
+	 dh_installchangelogs CHANGELOG.rst
 
 
-- 
cgit v1.2.3


From 68a946c5e13cf94181bfcebb98fc3ea93873e303 Mon Sep 17 00:00:00 2001
From: "Kali Kaneko (leap communications)" <kali@leap.se>
Date: Tue, 26 Apr 2016 23:57:34 -0400
Subject: [pkg] fix changelog path in manifest

---
 MANIFEST.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index d1a0e9a..8a66511 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,5 @@
 include pkg/*
 include versioneer.py
 include LICENSE
-include CHANGELOG
+include CHANGELOG.rst
 include src/leap/keymanager/_version.py
-- 
cgit v1.2.3


From 0aecef293fd86507da3565219982ce31c5cfde06 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Mon, 25 Apr 2016 20:24:48 -0300
Subject: [feat] creat an iterator to build dicts

Need pizza!!!!!!
---
 src/leap/keymanager/keys.py | 39 ++++++++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index 34d921c..1955d54 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -192,6 +192,11 @@ class EncryptionKey(object):
 
     __metaclass__ = ABCMeta
 
+    __slots__ = ('address', 'uids', 'fingerprint', 'key_data',
+                 'private', 'length', 'expiry_date', 'validation',
+                 'last_audited_at', 'refreshed_at',
+                 'encr_used', 'sign_used', '_index')
+
     def __init__(self, address=None, uids=[], fingerprint="",
                  key_data="", private=False, length=0, expiry_date=None,
                  validation=ValidationLevels.Weak_Chain, last_audited_at=None,
@@ -212,6 +217,7 @@ class EncryptionKey(object):
         self.refreshed_at = refreshed_at
         self.encr_used = encr_used
         self.sign_used = sign_used
+        self._index = len(self.__slots__)
 
     def get_json(self):
         """
@@ -258,19 +264,26 @@ class EncryptionKey(object):
             KEY_TAGS_KEY: [KEYMANAGER_ACTIVE_TAG],
         })
 
-    def get_dict(self):
-        """
-        :return: a serializable dict representation of this key.
-        :rtype: dict
-        """
-        return {
-          KEY_ADDRESS_KEY: self.address,
-          KEY_UIDS_KEY: self.uids,
-          KEY_FINGERPRINT_KEY: self.fingerprint,
-          KEY_LENGTH_KEY: self.length,
-          KEY_EXPIRY_DATE_KEY: str(self.expiry_date),
-          KEY_DATA_KEY: self.key_data
-        }
+    def next(self):
+        if self._index == 0:
+            self._index = len(self.__slots__)
+            raise StopIteration
+
+        self._index -= 1
+        key = self.__slots__[self._index]
+
+        if key.startswith('_'):
+            return self.next()
+
+        value = getattr(self, key)
+        if key == "validation":
+            value = str(value)
+        elif key in ["expiry_date", "last_audited_at", "refreshed_at"]:
+            value = str(value)
+        return key, value
+
+    def __iter__(self):
+        return self
 
     def __repr__(self):
         """
-- 
cgit v1.2.3


From 2d115d9dff46e81ddb10cf31f45d7a955e5658e4 Mon Sep 17 00:00:00 2001
From: "Kali Kaneko (leap communications)" <kali@leap.se>
Date: Wed, 4 May 2016 17:58:27 -0400
Subject: update changelog

---
 debian/changelog | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/debian/changelog b/debian/changelog
index de46a4d..e50cf44 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+leap-keymanager (0.5.1) unstable; urgency=medium
+
+  * Update to 0.5.1 release 
+
+ -- Ben Carrillo <ben@futeisha.org>  Wed, 04 May 2016 17:55:36 -0400
+
 leap-keymanager (0.5.0) unstable; urgency=medium
 
   * Update to 0.5.0 release. 
-- 
cgit v1.2.3