From a4872bd64884cccd85e63cc2cae631f26dbc96c4 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 27 Jan 2016 20:45:00 -0300
Subject: [feat] defer blocking requests calls to thread

That's a temporary fix for #6506
This commit adapts code to deal with deferreds coming from calling
requests from Twisted. Next step is just to change requests for twisted
http client present in leap.common.
Unfortunately, this last step will be a bit longer and would be better
to have integrations tests to ensure current HTTP behaviour.
---
 src/leap/keymanager/__init__.py | 42 +++++++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 16 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index c7886e0..06128c0 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -58,6 +58,7 @@ import logging
 import requests
 
 from twisted.internet import defer
+from twisted.internet import threads
 from urlparse import urlparse
 
 from leap.common.check import leap_assert
@@ -185,6 +186,7 @@ class KeyManager(object):
             lambda klass: klass.__name__ == ktype,
             self._wrapper_map).pop()
 
+    @defer.inlineCallbacks
     def _get(self, uri, data=None):
         """
         Send a GET request to C{uri} containing C{data}.
@@ -200,7 +202,8 @@ class KeyManager(object):
         leap_assert(
             self._ca_cert_path is not None,
             'We need the CA certificate path!')
-        res = self._fetcher.get(uri, data=data, verify=self._ca_cert_path)
+        res = yield threads.deferToThread(self._fetcher.get, uri, data=data,
+                                          verify=self._ca_cert_path)
         # Nickserver now returns 404 for key not found and 500 for
         # other cases (like key too small), so we are skipping this
         # check for the time being
@@ -211,7 +214,7 @@ class KeyManager(object):
         # leap_assert(
         #     res.headers['content-type'].startswith('application/json'),
         #     'Content-type is not JSON.')
-        return res
+        defer.returnValue(res)
 
     def _get_with_combined_ca_bundle(self, uri, data=None):
         """
@@ -228,9 +231,11 @@ class KeyManager(object):
         :return: The response to the request.
         :rtype: requests.Response
         """
-        return self._fetcher.get(
-            uri, data=data, verify=self._combined_ca_bundle)
+        return threads.deferToThread(self._fetcher.get,
+                                     uri, data=data,
+                                     verify=self._combined_ca_bundle)
 
+    @defer.inlineCallbacks
     def _put(self, uri, data=None):
         """
         Send a PUT request to C{uri} containing C{data}.
@@ -253,14 +258,17 @@ class KeyManager(object):
         leap_assert(
             self._token is not None,
             'We need a token to interact with webapp!')
-        res = self._fetcher.put(
-            uri, data=data, verify=self._ca_cert_path,
-            headers={'Authorization': 'Token token=%s' % self._token})
+        headers = {'Authorization': 'Token token=%s' % self._token}
+        res = yield threads.deferToThread(self._fetcher.put,
+                                          uri, data=data,
+                                          verify=self._ca_cert_path,
+                                          headers=headers)
         # assert that the response is valid
         res.raise_for_status()
-        return res
+        defer.returnValue(res)
 
     @memoized_method(invalidation=300)
+    @defer.inlineCallbacks
     def _fetch_keys_from_server(self, address):
         """
         Fetch keys bound to address from nickserver and insert them in
@@ -279,7 +287,7 @@ class KeyManager(object):
         d = defer.succeed(None)
         res = None
         try:
-            res = self._get(self._nickserver_uri, {'address': address})
+            res = yield self._get(self._nickserver_uri, {'address': address})
             res.raise_for_status()
             server_keys = res.json()
 
@@ -307,7 +315,7 @@ class KeyManager(object):
         except Exception as e:
             d = defer.fail(KeyNotFound(e.message))
             logger.warning("Error retrieving key: %r" % (e,))
-        return d
+        yield d
 
     #
     # key management
@@ -339,8 +347,9 @@ class KeyManager(object):
                 self._api_uri,
                 self._api_version,
                 self._uid)
-            self._put(uri, data)
+            d = self._put(uri, data)
             emit_async(catalog.KEYMANAGER_DONE_UPLOADING_KEYS, self._address)
+            return d
 
         d = self.get_key(
             self._address, ktype, private=False, fetch_remote=False)
@@ -822,6 +831,7 @@ class KeyManager(object):
             d.addCallback(lambda _: self.put_key(privkey, address))
         return d
 
+    @defer.inlineCallbacks
     def fetch_key(self, address, uri, ktype,
                   validation=ValidationLevels.Weak_Chain):
         """
@@ -852,20 +862,20 @@ class KeyManager(object):
 
         logger.info("Fetch key for %s from %s" % (address, uri))
         try:
-            res = self._get_with_combined_ca_bundle(uri)
+            res = yield self._get_with_combined_ca_bundle(uri)
         except Exception as e:
             logger.warning("There was a problem fetching key: %s" % (e,))
-            return defer.fail(KeyNotFound(uri))
+            raise KeyNotFound(uri)
         if not res.ok:
-            return defer.fail(KeyNotFound(uri))
+            raise KeyNotFound(uri)
 
         # XXX parse binary keys
         pubkey, _ = _keys.parse_ascii_key(res.content)
         if pubkey is None:
-            return defer.fail(KeyNotFound(uri))
+            raise KeyNotFound(uri)
 
         pubkey.validation = validation
-        return self.put_key(pubkey, address)
+        yield self.put_key(pubkey, address)
 
     def _assert_supported_key_type(self, ktype):
         """
-- 
cgit v1.2.3


From e759ab61e8a22235c45192750b536612a6c2cb8a Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 27 Jan 2016 21:35:43 -0300
Subject: [refactor] isolate requests

Isolate requests lib related code and update docstrings.
---
 src/leap/keymanager/__init__.py              | 112 +++++++++++++--------------
 src/leap/keymanager/tests/test_keymanager.py |   8 +-
 2 files changed, 56 insertions(+), 64 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 06128c0..6413e9e 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -187,35 +187,43 @@ class KeyManager(object):
             self._wrapper_map).pop()
 
     @defer.inlineCallbacks
-    def _get(self, uri, data=None):
+    def _get_json(self, address):
         """
         Send a GET request to C{uri} containing C{data}.
 
         :param uri: The URI of the request.
         :type uri: str
-        :param data: The body of the request.
-        :type data: dict, str or file
 
-        :return: The response to the request.
-        :rtype: requests.Response
+        :return: A deferred that will be fired with the GET response
+        :rtype: Deferred
         """
         leap_assert(
             self._ca_cert_path is not None,
             'We need the CA certificate path!')
-        res = yield threads.deferToThread(self._fetcher.get, uri, data=data,
-                                          verify=self._ca_cert_path)
-        # Nickserver now returns 404 for key not found and 500 for
-        # other cases (like key too small), so we are skipping this
-        # check for the time being
-        # res.raise_for_status()
-
+        try:
+            uri, data = self._nickserver_uri, {'address': address}
+            res = yield threads.deferToThread(self._fetcher.get, uri,
+                                              data=data,
+                                              verify=self._ca_cert_path)
+            res.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                raise KeyNotFound(address)
+            else:
+                raise KeyNotFound(e.message)
+            logger.warning("HTTP error retrieving key: %r" % (e,))
+            logger.warning("%s" % (res.content,))
+        except Exception as e:
+            raise KeyNotFound(e.message)
+            logger.warning("Error retrieving key: %r" % (e,))
         # Responses are now text/plain, although it's json anyway, but
         # this will fail when it shouldn't
         # leap_assert(
         #     res.headers['content-type'].startswith('application/json'),
         #     'Content-type is not JSON.')
-        defer.returnValue(res)
+        defer.returnValue(res.json())
 
+    @defer.inlineCallbacks
     def _get_with_combined_ca_bundle(self, uri, data=None):
         """
         Send a GET request to C{uri} containing C{data}.
@@ -228,12 +236,18 @@ class KeyManager(object):
         :param data: The body of the request.
         :type data: dict, str or file
 
-        :return: The response to the request.
-        :rtype: requests.Response
+        :return: A deferred that will be fired with the GET response
+        :rtype: Deferred
         """
-        return threads.deferToThread(self._fetcher.get,
-                                     uri, data=data,
-                                     verify=self._combined_ca_bundle)
+        try:
+            res = yield threads.deferToThread(self._fetcher.get, uri,
+                                              verify=self._combined_ca_bundle)
+        except Exception as e:
+            logger.warning("There was a problem fetching key: %s" % (e,))
+            raise KeyNotFound(uri)
+        if not res.ok:
+            raise KeyNotFound(uri)
+        defer.returnValue(res.content)
 
     @defer.inlineCallbacks
     def _put(self, uri, data=None):
@@ -249,8 +263,8 @@ class KeyManager(object):
         :param data: The body of the request.
         :type data: dict, str or file
 
-        :return: The response to the request.
-        :rtype: requests.Response
+        :return: A deferred that will be fired when PUT request finishes
+        :rtype: Deferred
         """
         leap_assert(
             self._ca_cert_path is not None,
@@ -284,38 +298,22 @@ class KeyManager(object):
 
         """
         # request keys from the nickserver
-        d = defer.succeed(None)
-        res = None
-        try:
-            res = yield self._get(self._nickserver_uri, {'address': address})
-            res.raise_for_status()
-            server_keys = res.json()
-
-            # insert keys in local database
-            if self.OPENPGP_KEY in server_keys:
-                # nicknym server is authoritative for its own domain,
-                # for other domains the key might come from key servers.
-                validation_level = ValidationLevels.Weak_Chain
-                _, domain = _split_email(address)
-                if (domain == _get_domain(self._nickserver_uri)):
-                    validation_level = ValidationLevels.Provider_Trust
-
-                d = self.put_raw_key(
-                    server_keys['openpgp'],
-                    OpenPGPKey,
-                    address=address,
-                    validation=validation_level)
-        except requests.exceptions.HTTPError as e:
-            if e.response.status_code == 404:
-                d = defer.fail(KeyNotFound(address))
-            else:
-                d = defer.fail(KeyNotFound(e.message))
-            logger.warning("HTTP error retrieving key: %r" % (e,))
-            logger.warning("%s" % (res.content,))
-        except Exception as e:
-            d = defer.fail(KeyNotFound(e.message))
-            logger.warning("Error retrieving key: %r" % (e,))
-        yield d
+        server_keys = yield self._get_json(address)
+
+        # insert keys in local database
+        if self.OPENPGP_KEY in server_keys:
+            # nicknym server is authoritative for its own domain,
+            # for other domains the key might come from key servers.
+            validation_level = ValidationLevels.Weak_Chain
+            _, domain = _split_email(address)
+            if (domain == _get_domain(self._nickserver_uri)):
+                validation_level = ValidationLevels.Provider_Trust
+
+            yield self.put_raw_key(
+                server_keys['openpgp'],
+                OpenPGPKey,
+                address=address,
+                validation=validation_level)
 
     #
     # key management
@@ -861,16 +859,10 @@ class KeyManager(object):
         _keys = self._wrapper_map[ktype]
 
         logger.info("Fetch key for %s from %s" % (address, uri))
-        try:
-            res = yield self._get_with_combined_ca_bundle(uri)
-        except Exception as e:
-            logger.warning("There was a problem fetching key: %s" % (e,))
-            raise KeyNotFound(uri)
-        if not res.ok:
-            raise KeyNotFound(uri)
+        ascii_content = yield self._get_with_combined_ca_bundle(uri)
 
         # XXX parse binary keys
-        pubkey, _ = _keys.parse_ascii_key(res.content)
+        pubkey, _ = _keys.parse_ascii_key(ascii_content)
         if pubkey is None:
             raise KeyNotFound(uri)
 
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index 856d6da..afcbd5f 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -342,7 +342,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL, data=None,
+        get_mock.assert_called_once_with(REMOTE_KEY_URL,
                                          verify=ca_bundle.where())
 
     @inlineCallbacks
@@ -353,7 +353,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL, data=None,
+        get_mock.assert_called_once_with(REMOTE_KEY_URL,
                                          verify=ca_bundle.where())
 
     @inlineCallbacks
@@ -364,7 +364,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL, data=None,
+        get_mock.assert_called_once_with(REMOTE_KEY_URL,
                                          verify=ca_bundle.where())
 
     @inlineCallbacks
@@ -383,7 +383,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
                 yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
                 # assert that combined bundle file is passed to get call
-                get_mock.assert_called_once_with(REMOTE_KEY_URL, data=None,
+                get_mock.assert_called_once_with(REMOTE_KEY_URL,
                                                  verify=tmp_output.name)
 
                 # assert that files got appended
-- 
cgit v1.2.3


From cb32761d3dc97956085cd34a8f1e2e06432e8141 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Wed, 27 Jan 2016 23:18:04 -0300
Subject: [feat] use HTTPClient instead of requests

This commit adapts code to use HTTPClient instead of requests.
requests library receives a certificate as parameter during requests
while HTTPClient recelives a cert only on constructor. In order to have
both types (leap cert and commercial certs) working together we
introduced two clients on constructor.
---
 src/leap/keymanager/__init__.py              |  63 +++++++++-------
 src/leap/keymanager/tests/test_keymanager.py | 109 +++++++++++----------------
 2 files changed, 82 insertions(+), 90 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 6413e9e..1dcf642 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -22,6 +22,8 @@ import fileinput
 import os
 import sys
 import tempfile
+import json
+import urllib
 
 from leap.common import ca_bundle
 
@@ -58,10 +60,10 @@ import logging
 import requests
 
 from twisted.internet import defer
-from twisted.internet import threads
 from urlparse import urlparse
 
 from leap.common.check import leap_assert
+from leap.common.http import HTTPClient
 from leap.common.events import emit_async, catalog
 from leap.common.decorators import memoized_method
 
@@ -142,6 +144,8 @@ class KeyManager(object):
         # the following are used to perform https requests
         self._fetcher = requests
         self._combined_ca_bundle = self._create_combined_bundle_file()
+        self._async_client = HTTPClient(self._combined_ca_bundle)
+        self._async_client_pinned = HTTPClient(self._ca_cert_path)
 
     #
     # destructor
@@ -201,27 +205,29 @@ class KeyManager(object):
             self._ca_cert_path is not None,
             'We need the CA certificate path!')
         try:
-            uri, data = self._nickserver_uri, {'address': address}
-            res = yield threads.deferToThread(self._fetcher.get, uri,
-                                              data=data,
-                                              verify=self._ca_cert_path)
-            res.raise_for_status()
-        except requests.exceptions.HTTPError as e:
-            if e.response.status_code == 404:
-                raise KeyNotFound(address)
-            else:
-                raise KeyNotFound(e.message)
+            uri = self._nickserver_uri + '?address=' + address
+            content = yield self._async_client_pinned.request(str(uri), 'GET')
+            json_content = json.loads(content)
+        except IOError as e:
+            # FIXME: 404 doesnt raise today, but it wont produce json anyway
+            # if e.response.status_code == 404:
+                # raise KeyNotFound(address)
             logger.warning("HTTP error retrieving key: %r" % (e,))
-            logger.warning("%s" % (res.content,))
+            logger.warning("%s" % (content,))
+            raise KeyNotFound(e), None, sys.exc_info()[2]
+        except ValueError as v:
+            logger.warning("Invalid JSON data from key: %s" % (uri,))
+            raise KeyNotFound(v.message + ' - ' + uri), None, sys.exc_info()[2]
+
         except Exception as e:
-            raise KeyNotFound(e.message)
             logger.warning("Error retrieving key: %r" % (e,))
+            raise KeyNotFound(e.message), None, sys.exc_info()[2]
         # Responses are now text/plain, although it's json anyway, but
         # this will fail when it shouldn't
         # leap_assert(
         #     res.headers['content-type'].startswith('application/json'),
         #     'Content-type is not JSON.')
-        defer.returnValue(res.json())
+        defer.returnValue(json_content)
 
     @defer.inlineCallbacks
     def _get_with_combined_ca_bundle(self, uri, data=None):
@@ -240,14 +246,13 @@ class KeyManager(object):
         :rtype: Deferred
         """
         try:
-            res = yield threads.deferToThread(self._fetcher.get, uri,
-                                              verify=self._combined_ca_bundle)
+            content = yield self._async_client.request(str(uri), 'GET')
         except Exception as e:
             logger.warning("There was a problem fetching key: %s" % (e,))
             raise KeyNotFound(uri)
-        if not res.ok:
+        if not content:
             raise KeyNotFound(uri)
-        defer.returnValue(res.content)
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     def _put(self, uri, data=None):
@@ -272,14 +277,20 @@ class KeyManager(object):
         leap_assert(
             self._token is not None,
             'We need a token to interact with webapp!')
-        headers = {'Authorization': 'Token token=%s' % self._token}
-        res = yield threads.deferToThread(self._fetcher.put,
-                                          uri, data=data,
-                                          verify=self._ca_cert_path,
-                                          headers=headers)
-        # assert that the response is valid
-        res.raise_for_status()
-        defer.returnValue(res)
+        if type(data) == dict:
+            data = urllib.urlencode(data)
+        headers = {'Authorization': [str('Token token=%s' % self._token)]}
+        headers['Content-Type'] = ['application/x-www-form-urlencoded']
+        try:
+            res = yield self._async_client_pinned.request(str(uri), 'PUT', body=str(data), headers=headers)
+        except Exception as e:
+            logger.warning("Error uploading key: %r" % (e,))
+            raise e
+        if 'error' in res:
+            # FIXME: That's a workaround for 500,
+            # we need to implement a readBody to assert response code
+            logger.warning("Error uploading key: %r" % (res,))
+            raise Exception(res)
 
     @memoized_method(invalidation=300)
     @defer.inlineCallbacks
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index afcbd5f..c3426b6 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -21,11 +21,14 @@ Tests for the Key Manager.
 """
 
 from os import path
+import json
+import urllib
 from datetime import datetime
 import tempfile
+import pkg_resources
 from leap.common import ca_bundle
 from mock import Mock, MagicMock, patch
-from twisted.internet.defer import inlineCallbacks
+from twisted.internet import defer
 from twisted.trial import unittest
 
 from leap.keymanager import (
@@ -127,7 +130,7 @@ class KeyManagerUtilTestCase(unittest.TestCase):
 
 class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_all_keys_in_db(self):
         km = self._key_manager()
         yield km._wrapper_map[OpenPGPKey].put_ascii_key(PRIVATE_KEY, ADDRESS)
@@ -142,7 +145,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         self.assertTrue(ADDRESS in keys[0].address)
         self.assertTrue(keys[0].private)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_public_key(self):
         km = self._key_manager()
         yield km._wrapper_map[OpenPGPKey].put_ascii_key(PRIVATE_KEY, ADDRESS)
@@ -155,7 +158,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
             key.fingerprint.lower(), KEY_FINGERPRINT.lower())
         self.assertFalse(key.private)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_private_key(self):
         km = self._key_manager()
         yield km._wrapper_map[OpenPGPKey].put_ascii_key(PRIVATE_KEY, ADDRESS)
@@ -173,7 +176,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         d = km.send_key(OpenPGPKey)
         return self.assertFailure(d, KeyNotFound)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_send_key(self):
         """
         Test that request is well formed when sending keys to server.
@@ -181,7 +184,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         token = "mytoken"
         km = self._key_manager(token=token)
         yield km._wrapper_map[OpenPGPKey].put_ascii_key(PUBLIC_KEY, ADDRESS)
-        km._fetcher.put = Mock()
+        km._async_client_pinned.request = Mock(return_value=defer.succeed(''))
         # the following data will be used on the send
         km.ca_cert_path = 'capath'
         km.session_id = 'sessionid'
@@ -191,13 +194,15 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         yield km.send_key(OpenPGPKey)
         # setup expected args
         pubkey = yield km.get_key(km._address, OpenPGPKey)
-        data = {
+        data = urllib.urlencode({
             km.PUBKEY_KEY: pubkey.key_data,
-        }
+        })
+        headers = {'Authorization': [str('Token token=%s' % token)]}
+        headers['Content-Type'] = ['application/x-www-form-urlencoded']
         url = '%s/%s/users/%s.json' % ('apiuri', 'apiver', 'myuid')
-        km._fetcher.put.assert_called_once_with(
-            url, data=data, verify='capath',
-            headers={'Authorization': 'Token token=%s' % token},
+        km._async_client_pinned.request.assert_called_once_with(
+            str(url), 'PUT', body=str(data),
+            headers=headers
         )
 
     def test_fetch_keys_from_server(self):
@@ -205,19 +210,19 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         Test that the request is well formed when fetching keys from server.
         """
         km = self._key_manager(url=NICKSERVER_URI)
+        expected_url = NICKSERVER_URI + '?address=' + ADDRESS_2
 
         def verify_the_call(_):
-            km._fetcher.get.assert_called_once_with(
-                NICKSERVER_URI,
-                data={'address': ADDRESS_2},
-                verify='cacertpath',
+            km._async_client_pinned.request.assert_called_once_with(
+                expected_url,
+                'GET',
             )
 
         d = self._fetch_key(km, ADDRESS_2, PUBLIC_KEY_2)
         d.addCallback(verify_the_call)
         return d
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_key_fetches_from_server(self):
         """
         Test that getting a key successfuly fetches from server.
@@ -229,7 +234,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         self.assertTrue(ADDRESS in key.address)
         self.assertEqual(key.validation, ValidationLevels.Provider_Trust)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_get_key_fetches_other_domain(self):
         """
         Test that getting a key successfuly fetches from server.
@@ -245,18 +250,10 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         """
         :returns: a Deferred that will fire with the OpenPGPKey
         """
-        class Response(object):
-            status_code = 200
-            headers = {'content-type': 'application/json'}
-
-            def json(self):
-                return {'address': address, 'openpgp': key}
-
-            def raise_for_status(self):
-                pass
+        data = json.dumps({'address': address, 'openpgp': key})
 
         # mock the fetcher so it returns the key for ADDRESS_2
-        km._fetcher.get = Mock(return_value=Response())
+        km._async_client_pinned.request = Mock(return_value=defer.succeed(data))
         km.ca_cert_path = 'cacertpath'
         # try to key get without fetching from server
         d_fail = km.get_key(address, OpenPGPKey, fetch_remote=False)
@@ -265,7 +262,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         d.addCallback(lambda _: km.get_key(address, OpenPGPKey))
         return d
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_put_key_ascii(self):
         """
         Test that putting ascii key works
@@ -277,7 +274,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         self.assertIsInstance(key, OpenPGPKey)
         self.assertTrue(ADDRESS in key.address)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_uri_ascii_key(self):
         """
         Test that fetch key downloads the ascii key and gets included in
@@ -285,11 +282,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         """
         km = self._key_manager()
 
-        class Response(object):
-            ok = True
-            content = PUBLIC_KEY
-
-        km._fetcher.get = Mock(return_value=Response())
+        km._async_client.request = Mock(return_value=defer.succeed(PUBLIC_KEY))
 
         yield km.fetch_key(ADDRESS, "http://site.domain/key", OpenPGPKey)
         key = yield km.get_key(ADDRESS, OpenPGPKey)
@@ -316,25 +309,16 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         """
         km = self._key_manager()
 
-        class Response(object):
-            ok = True
-            content = PUBLIC_KEY
-
-        km._fetcher.get = Mock(return_value=Response())
+        km._async_client.request = Mock(return_value=defer.succeed(PUBLIC_KEY))
         d = km.fetch_key(ADDRESS_2, "http://site.domain/key", OpenPGPKey)
         return self.assertFailure(d, KeyAddressMismatch)
 
     def _mock_get_response(self, km, body):
-        class Response(object):
-            ok = True
-            content = body
-
-        mock = MagicMock(return_value=Response())
-        km._fetcher.get = mock
+        km._async_client.request = MagicMock(return_value=defer.succeed(body))
 
-        return mock
+        return km._async_client.request
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_key_uses_ca_bundle_if_none_specified(self):
         ca_cert_path = None
         km = self._key_manager(ca_cert_path=ca_cert_path)
@@ -342,10 +326,9 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL,
-                                         verify=ca_bundle.where())
+        get_mock.assert_called_once_with(REMOTE_KEY_URL, 'GET')
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_key_uses_ca_bundle_if_empty_string_specified(self):
         ca_cert_path = ''
         km = self._key_manager(ca_cert_path=ca_cert_path)
@@ -353,10 +336,9 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL,
-                                         verify=ca_bundle.where())
+        get_mock.assert_called_once_with(REMOTE_KEY_URL, 'GET')
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_key_use_default_ca_bundle_if_set_as_ca_cert_path(self):
         ca_cert_path = ca_bundle.where()
         km = self._key_manager(ca_cert_path=ca_cert_path)
@@ -364,14 +346,14 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
-        get_mock.assert_called_once_with(REMOTE_KEY_URL,
-                                         verify=ca_bundle.where())
+        get_mock.assert_called_once_with(REMOTE_KEY_URL, 'GET')
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_fetch_uses_combined_ca_bundle_otherwise(self):
         with tempfile.NamedTemporaryFile() as tmp_input, \
                 tempfile.NamedTemporaryFile(delete=False) as tmp_output:
-            ca_content = 'some\ncontent\n'
+            ca_content = pkg_resources.resource_string('leap.common.testing',
+                                                       'cacert.pem')
             ca_cert_path = tmp_input.name
             self._dump_to_file(ca_cert_path, ca_content)
 
@@ -383,8 +365,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
                 yield km.fetch_key(ADDRESS_OTHER, REMOTE_KEY_URL, OpenPGPKey)
 
                 # assert that combined bundle file is passed to get call
-                get_mock.assert_called_once_with(REMOTE_KEY_URL,
-                                                 verify=tmp_output.name)
+                get_mock.assert_called_once_with(REMOTE_KEY_URL, 'GET')
 
                 # assert that files got appended
                 expected = self._slurp_file(ca_bundle.where()) + ca_content
@@ -402,7 +383,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
             content = f.read()
         return content
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_decrypt_updates_sign_used_for_signer(self):
         # given
         km = self._key_manager()
@@ -420,7 +401,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         # then
         self.assertEqual(True, key.sign_used)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_decrypt_does_not_update_sign_used_for_recipient(self):
         # given
         km = self._key_manager()
@@ -445,7 +426,7 @@ class KeyManagerCryptoTestCase(KeyManagerWithSoledadTestCase):
 
     RAW_DATA = 'data'
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_keymanager_openpgp_encrypt_decrypt(self):
         km = self._key_manager()
         # put raw private key
@@ -464,7 +445,7 @@ class KeyManagerCryptoTestCase(KeyManagerWithSoledadTestCase):
                                fetch_remote=False)
         self.assertEqual(signingkey.fingerprint, key.fingerprint)
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_keymanager_openpgp_encrypt_decrypt_wrong_sign(self):
         km = self._key_manager()
         # put raw keys
@@ -481,7 +462,7 @@ class KeyManagerCryptoTestCase(KeyManagerWithSoledadTestCase):
         self.assertEqual(self.RAW_DATA, rawdata)
         self.assertTrue(isinstance(signingkey, errors.InvalidSignature))
 
-    @inlineCallbacks
+    @defer.inlineCallbacks
     def test_keymanager_openpgp_sign_verify(self):
         km = self._key_manager()
         # put raw private keys
-- 
cgit v1.2.3


From 00c0ed0d1b14aff8be0172e03bf26e1c30477af6 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Tue, 2 Feb 2016 19:51:36 -0300
Subject: [docs] add docstrings and fixes pep8

Some methods were missing docstrings and some code was exceeding the 80
column limit. Also some asserts arent needed anymore.
---
 src/leap/keymanager/__init__.py | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 1dcf642..7e4d30e 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -184,26 +184,29 @@ class KeyManager(object):
 
     def _key_class_from_type(self, ktype):
         """
-        Return key class from string representation of key type.
+        Given a class type, return a class
+
+        :param ktype: string representation of a class name
+        :type ktype: str
+
+        :return: A class with the matching name
+        :rtype: classobj or type
         """
         return filter(
             lambda klass: klass.__name__ == ktype,
             self._wrapper_map).pop()
 
     @defer.inlineCallbacks
-    def _get_json(self, address):
+    def _get_key_from_nicknym(self, address):
         """
         Send a GET request to C{uri} containing C{data}.
 
-        :param uri: The URI of the request.
-        :type uri: str
+        :param address: The URI of the request.
+        :type address: str
 
-        :return: A deferred that will be fired with the GET response
+        :return: A deferred that will be fired with GET content as json (dict)
         :rtype: Deferred
         """
-        leap_assert(
-            self._ca_cert_path is not None,
-            'We need the CA certificate path!')
         try:
             uri = self._nickserver_uri + '?address=' + address
             content = yield self._async_client_pinned.request(str(uri), 'GET')
@@ -214,7 +217,7 @@ class KeyManager(object):
                 # raise KeyNotFound(address)
             logger.warning("HTTP error retrieving key: %r" % (e,))
             logger.warning("%s" % (content,))
-            raise KeyNotFound(e), None, sys.exc_info()[2]
+            raise KeyNotFound(e.message), None, sys.exc_info()[2]
         except ValueError as v:
             logger.warning("Invalid JSON data from key: %s" % (uri,))
             raise KeyNotFound(v.message + ' - ' + uri), None, sys.exc_info()[2]
@@ -271,9 +274,6 @@ class KeyManager(object):
         :return: A deferred that will be fired when PUT request finishes
         :rtype: Deferred
         """
-        leap_assert(
-            self._ca_cert_path is not None,
-            'We need the CA certificate path!')
         leap_assert(
             self._token is not None,
             'We need a token to interact with webapp!')
@@ -282,7 +282,9 @@ class KeyManager(object):
         headers = {'Authorization': [str('Token token=%s' % self._token)]}
         headers['Content-Type'] = ['application/x-www-form-urlencoded']
         try:
-            res = yield self._async_client_pinned.request(str(uri), 'PUT', body=str(data), headers=headers)
+            res = yield self._async_client_pinned.request(str(uri), 'PUT',
+                                                          body=str(data),
+                                                          headers=headers)
         except Exception as e:
             logger.warning("Error uploading key: %r" % (e,))
             raise e
@@ -309,7 +311,7 @@ class KeyManager(object):
 
         """
         # request keys from the nickserver
-        server_keys = yield self._get_json(address)
+        server_keys = yield self._get_key_from_nicknym(address)
 
         # insert keys in local database
         if self.OPENPGP_KEY in server_keys:
@@ -357,7 +359,9 @@ class KeyManager(object):
                 self._api_version,
                 self._uid)
             d = self._put(uri, data)
-            emit_async(catalog.KEYMANAGER_DONE_UPLOADING_KEYS, self._address)
+            d.addCallback(lambda _:
+                          emit_async(catalog.KEYMANAGER_DONE_UPLOADING_KEYS,
+                                     self._address))
             return d
 
         d = self.get_key(
-- 
cgit v1.2.3


From 4e13b7595fc5c8e245244eb535525ae8333ef9dc Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Tue, 9 Feb 2016 16:32:53 +0100
Subject: [style] fix pep8

---
 src/leap/keymanager/tests/test_keymanager.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index c3426b6..0f08326 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -253,7 +253,8 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         data = json.dumps({'address': address, 'openpgp': key})
 
         # mock the fetcher so it returns the key for ADDRESS_2
-        km._async_client_pinned.request = Mock(return_value=defer.succeed(data))
+        km._async_client_pinned.request = Mock(
+            return_value=defer.succeed(data))
         km.ca_cert_path = 'cacertpath'
         # try to key get without fetching from server
         d_fail = km.get_key(address, OpenPGPKey, fetch_remote=False)
-- 
cgit v1.2.3


From c18b69441e24c57a3130e27356edc0b9395d78f8 Mon Sep 17 00:00:00 2001
From: Victor Shyba <victor.shyba@gmail.com>
Date: Thu, 11 Feb 2016 19:22:34 -0300
Subject: [feat] defer decrypt, gen_key and encrypt

This commit put those gnupg operations to be run on external threads
limited by the amount of cores present on user machine.
Some gnupg calls spawn processes and communicating to them is a
synchronous operation, so running outside of a reactor should improve
response time by avoiding reactor locking.
---
 src/leap/keymanager/__init__.py           | 18 +++++++-------
 src/leap/keymanager/openpgp.py            | 40 ++++++++++++++++++++++---------
 src/leap/keymanager/tests/test_openpgp.py | 30 +++++++++++------------
 3 files changed, 53 insertions(+), 35 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 7e4d30e..9aa7139 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -572,15 +572,15 @@ class KeyManager(object):
         self._assert_supported_key_type(ktype)
         _keys = self._wrapper_map[ktype]
 
+        @defer.inlineCallbacks
         def encrypt(keys):
             pubkey, signkey = keys
-            encrypted = _keys.encrypt(
+            encrypted = yield _keys.encrypt(
                 data, pubkey, passphrase, sign=signkey,
                 cipher_algo=cipher_algo)
             pubkey.encr_used = True
-            d = _keys.put_key(pubkey, address)
-            d.addCallback(lambda _: encrypted)
-            return d
+            yield _keys.put_key(pubkey, address)
+            defer.returnValue(encrypted)
 
         dpub = self.get_key(address, ktype, private=False,
                             fetch_remote=fetch_remote)
@@ -625,9 +625,10 @@ class KeyManager(object):
         self._assert_supported_key_type(ktype)
         _keys = self._wrapper_map[ktype]
 
+        @defer.inlineCallbacks
         def decrypt(keys):
             pubkey, privkey = keys
-            decrypted, signed = _keys.decrypt(
+            decrypted, signed = yield _keys.decrypt(
                 data, privkey, passphrase=passphrase, verify=pubkey)
             if pubkey is None:
                 signature = KeyNotFound(verify)
@@ -635,14 +636,13 @@ class KeyManager(object):
                 signature = pubkey
                 if not pubkey.sign_used:
                     pubkey.sign_used = True
-                    d = _keys.put_key(pubkey, verify)
-                    d.addCallback(lambda _: (decrypted, signature))
-                    return d
+                    yield _keys.put_key(pubkey, verify)
+                    defer.returnValue((decrypted, signature))
             else:
                 signature = InvalidSignature(
                     'Failed to verify signature with key %s' %
                     (pubkey.key_id,))
-            return (decrypted, signature)
+            defer.returnValue((decrypted, signature))
 
         dpriv = self.get_key(address, ktype, private=True)
         dpub = defer.succeed(None)
diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index d648137..9064043 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -27,9 +27,11 @@ import io
 
 
 from datetime import datetime
+from multiprocessing import cpu_count
 from gnupg import GPG
 from gnupg.gnupg import GPGUtilities
 from twisted.internet import defer
+from twisted.internet.threads import deferToThread
 
 from leap.common.check import leap_assert, leap_assert_type, leap_check
 from leap.keymanager import errors
@@ -55,6 +57,16 @@ logger = logging.getLogger(__name__)
 # A temporary GPG keyring wrapped to provide OpenPGP functionality.
 #
 
+# This function will be used to call blocking GPG functions outside
+# of Twisted reactor and match the concurrent calls to the amount of CPU cores
+cpu_core_semaphore = defer.DeferredSemaphore(cpu_count())
+
+
+def from_thread(func, *args, **kwargs):
+    call = lambda: deferToThread(func, *args, **kwargs)
+    return cpu_core_semaphore.run(call)
+
+
 class TempGPGWrapper(object):
     """
     A context manager that wraps a temporary GPG keyring which only contains
@@ -253,6 +265,7 @@ class OpenPGPScheme(EncryptionScheme):
         # make sure the key does not already exist
         leap_assert(is_address(address), 'Not an user address: %s' % address)
 
+        @defer.inlineCallbacks
         def _gen_key(_):
             with TempGPGWrapper(gpgbinary=self._gpgbinary) as gpg:
                 # TODO: inspect result, or use decorator
@@ -264,7 +277,7 @@ class OpenPGPScheme(EncryptionScheme):
                     name_comment='')
                 logger.info("About to generate keys... "
                             "This might take SOME time.")
-                gpg.gen_key(params)
+                yield from_thread(gpg.gen_key, params)
                 logger.info("Keys for %s have been successfully "
                             "generated." % (address,))
                 pubkeys = gpg.list_keys()
@@ -293,7 +306,7 @@ class OpenPGPScheme(EncryptionScheme):
                         gpg.export_keys(key['fingerprint'], secret=secret))
                     d = self.put_key(openpgp_key, address)
                     deferreds.append(d)
-                return defer.gatherResults(deferreds)
+                yield defer.gatherResults(deferreds)
 
         def key_already_exists(_):
             raise errors.KeyAlreadyExists(address)
@@ -686,6 +699,7 @@ class OpenPGPScheme(EncryptionScheme):
             raise errors.GPGError(
                 'Failed to encrypt/decrypt: %s' % stderr)
 
+    @defer.inlineCallbacks
     def encrypt(self, data, pubkey, passphrase=None, sign=None,
                 cipher_algo='AES256'):
         """
@@ -700,8 +714,8 @@ class OpenPGPScheme(EncryptionScheme):
         :param cipher_algo: The cipher algorithm to use.
         :type cipher_algo: str
 
-        :return: The encrypted data.
-        :rtype: str
+        :return: A Deferred that will be fired with the encrypted data.
+        :rtype: defer.Deferred
 
         :raise EncryptError: Raised if failed encrypting for some reason.
         """
@@ -713,7 +727,8 @@ class OpenPGPScheme(EncryptionScheme):
             leap_assert(sign.private is True)
             keys.append(sign)
         with TempGPGWrapper(keys, self._gpgbinary) as gpg:
-            result = gpg.encrypt(
+            result = yield from_thread(
+                gpg.encrypt,
                 data, pubkey.fingerprint,
                 default_key=sign.key_id if sign else None,
                 passphrase=passphrase, symmetric=False,
@@ -724,11 +739,12 @@ class OpenPGPScheme(EncryptionScheme):
             # result.data  - (bool) contains the result of the operation
             try:
                 self._assert_gpg_result_ok(result)
-                return result.data
+                defer.returnValue(result.data)
             except errors.GPGError as e:
                 logger.error('Failed to decrypt: %s.' % str(e))
                 raise errors.EncryptError()
 
+    @defer.inlineCallbacks
     def decrypt(self, data, privkey, passphrase=None, verify=None):
         """
         Decrypt C{data} using private @{privkey} and verify with C{verify} key.
@@ -743,8 +759,9 @@ class OpenPGPScheme(EncryptionScheme):
         :param verify: The key used to verify a signature.
         :type verify: OpenPGPKey
 
-        :return: The decrypted data and if signature verifies
-        :rtype: (unicode, bool)
+        :return: Deferred that will fire with the decrypted data and
+                 if signature verifies (unicode, bool)
+        :rtype: Deferred
 
         :raise DecryptError: Raised if failed decrypting for some reason.
         """
@@ -756,8 +773,9 @@ class OpenPGPScheme(EncryptionScheme):
             keys.append(verify)
         with TempGPGWrapper(keys, self._gpgbinary) as gpg:
             try:
-                result = gpg.decrypt(
-                    data, passphrase=passphrase, always_trust=True)
+                result = yield from_thread(gpg.decrypt,
+                                           data, passphrase=passphrase,
+                                           always_trust=True)
                 self._assert_gpg_result_ok(result)
 
                 # verify signature
@@ -767,7 +785,7 @@ class OpenPGPScheme(EncryptionScheme):
                         verify.fingerprint == result.pubkey_fingerprint):
                     sign_valid = True
 
-                return (result.data, sign_valid)
+                defer.returnValue((result.data, sign_valid))
             except errors.GPGError as e:
                 logger.error('Failed to decrypt: %s.' % str(e))
                 raise errors.DecryptError(str(e))
diff --git a/src/leap/keymanager/tests/test_openpgp.py b/src/leap/keymanager/tests/test_openpgp.py
index bae83db..96b40a0 100644
--- a/src/leap/keymanager/tests/test_openpgp.py
+++ b/src/leap/keymanager/tests/test_openpgp.py
@@ -109,7 +109,7 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         # encrypt
         yield pgp.put_ascii_key(PUBLIC_KEY, ADDRESS)
         pubkey = yield pgp.get_key(ADDRESS, private=False)
-        cyphertext = pgp.encrypt(data, pubkey)
+        cyphertext = yield pgp.encrypt(data, pubkey)
 
         self.assertTrue(cyphertext is not None)
         self.assertTrue(cyphertext != '')
@@ -121,7 +121,7 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         yield self._assert_key_not_found(pgp, ADDRESS, private=True)
         yield pgp.put_ascii_key(PRIVATE_KEY, ADDRESS)
         privkey = yield pgp.get_key(ADDRESS, private=True)
-        decrypted, _ = pgp.decrypt(cyphertext, privkey)
+        decrypted, _ = yield pgp.decrypt(cyphertext, privkey)
         self.assertEqual(decrypted, data)
 
         yield pgp.delete_key(pubkey)
@@ -171,9 +171,9 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         yield pgp.put_ascii_key(PRIVATE_KEY, ADDRESS)
         privkey = yield pgp.get_key(ADDRESS, private=True)
         pubkey = yield pgp.get_key(ADDRESS, private=False)
-        self.assertRaises(
-            AssertionError,
-            pgp.encrypt, data, privkey, sign=pubkey)
+        self.failureResultOf(
+            pgp.encrypt(data, privkey, sign=pubkey),
+            AssertionError)
 
     @inlineCallbacks
     def test_decrypt_verify_with_private_raises(self):
@@ -183,12 +183,11 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         yield pgp.put_ascii_key(PRIVATE_KEY, ADDRESS)
         privkey = yield pgp.get_key(ADDRESS, private=True)
         pubkey = yield pgp.get_key(ADDRESS, private=False)
-        encrypted_and_signed = pgp.encrypt(
+        encrypted_and_signed = yield pgp.encrypt(
             data, pubkey, sign=privkey)
-        self.assertRaises(
-            AssertionError,
-            pgp.decrypt,
-            encrypted_and_signed, privkey, verify=privkey)
+        self.failureResultOf(
+            pgp.decrypt(encrypted_and_signed, privkey, verify=privkey),
+            AssertionError)
 
     @inlineCallbacks
     def test_decrypt_verify_with_wrong_key(self):
@@ -198,11 +197,12 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         yield pgp.put_ascii_key(PRIVATE_KEY, ADDRESS)
         privkey = yield pgp.get_key(ADDRESS, private=True)
         pubkey = yield pgp.get_key(ADDRESS, private=False)
-        encrypted_and_signed = pgp.encrypt(data, pubkey, sign=privkey)
+        encrypted_and_signed = yield pgp.encrypt(data, pubkey, sign=privkey)
         yield pgp.put_ascii_key(PUBLIC_KEY_2, ADDRESS_2)
         wrongkey = yield pgp.get_key(ADDRESS_2)
-        decrypted, validsign = pgp.decrypt(encrypted_and_signed, privkey,
-                                           verify=wrongkey)
+        decrypted, validsign = yield pgp.decrypt(encrypted_and_signed,
+                                                 privkey,
+                                                 verify=wrongkey)
         self.assertEqual(decrypted, data)
         self.assertFalse(validsign)
 
@@ -232,9 +232,9 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         privkey2 = yield pgp.get_key(ADDRESS_2, private=True)
 
         data = 'data'
-        encrypted_and_signed = pgp.encrypt(
+        encrypted_and_signed = yield pgp.encrypt(
             data, pubkey2, sign=privkey)
-        res, validsign = pgp.decrypt(
+        res, validsign = yield pgp.decrypt(
             encrypted_and_signed, privkey2, verify=pubkey)
         self.assertEqual(data, res)
         self.assertTrue(validsign)
-- 
cgit v1.2.3


From 64eaf2ee426623072bc2d9b1faf77ab831cb3be1 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Tue, 15 Dec 2015 13:29:44 +0100
Subject: [feat] move validation, usage and audited date to the active document

- Resolves: #7485
---
 src/leap/keymanager/__init__.py              |   1 +
 src/leap/keymanager/keys.py                  | 132 ++++++++++---
 src/leap/keymanager/openpgp.py               | 268 +++++++++++----------------
 src/leap/keymanager/tests/test_keymanager.py |  15 +-
 src/leap/keymanager/tests/test_openpgp.py    |  54 +++++-
 src/leap/keymanager/tests/test_validation.py | 164 ++++++++++++++--
 6 files changed, 430 insertions(+), 204 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 9aa7139..8b3487f 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -439,6 +439,7 @@ class KeyManager(object):
         :return: A Deferred which fires with a list of all keys in local db.
         :rtype: Deferred
         """
+        # TODO: should it be based on activedocs?
         def build_keys(docs):
             return map(
                 lambda doc: build_key_from_dict(
diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index 91559c2..0de6a82 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -28,6 +28,7 @@ except ImportError:
 import logging
 import re
 import time
+import traceback
 
 
 from abc import ABCMeta, abstractmethod
@@ -110,39 +111,48 @@ def is_address(address):
     return bool(re.match('[\w.-]+@[\w.-]+', address))
 
 
-def build_key_from_dict(kClass, kdict):
+def build_key_from_dict(kClass, key, active=None):
     """
     Build an C{kClass} key based on info in C{kdict}.
 
-    :param kdict: Dictionary with key data.
-    :type kdict: dict
+    :param key: Dictionary with key data.
+    :type key: dict
+    :param active: Dictionary with active data.
+    :type active: dict
     :return: An instance of the key.
     :rtype: C{kClass}
     """
-    try:
-        validation = ValidationLevels.get(kdict[KEY_VALIDATION_KEY])
-    except ValueError:
-        logger.error("Not valid validation level (%s) for key %s",
-                     (kdict[KEY_VALIDATION_KEY], kdict[KEY_ID_KEY]))
-        validation = ValidationLevels.Weak_Chain
-
-    expiry_date = _to_datetime(kdict[KEY_EXPIRY_DATE_KEY])
-    last_audited_at = _to_datetime(kdict[KEY_LAST_AUDITED_AT_KEY])
-    refreshed_at = _to_datetime(kdict[KEY_REFRESHED_AT_KEY])
+    validation = ValidationLevels.Weak_Chain
+    last_audited_at = None
+    encr_used = False
+    sign_used = False
+
+    if active:
+        try:
+            validation = ValidationLevels.get(active[KEY_VALIDATION_KEY])
+        except ValueError:
+            logger.error("Not valid validation level (%s) for key %s",
+                         (active[KEY_VALIDATION_KEY], active[KEY_ID_KEY]))
+        last_audited_at = _to_datetime(active[KEY_LAST_AUDITED_AT_KEY])
+        encr_used = active[KEY_ENCR_USED_KEY]
+        sign_used = active[KEY_SIGN_USED_KEY]
+
+    expiry_date = _to_datetime(key[KEY_EXPIRY_DATE_KEY])
+    refreshed_at = _to_datetime(key[KEY_REFRESHED_AT_KEY])
 
     return kClass(
-        kdict[KEY_ADDRESS_KEY],
-        key_id=kdict[KEY_ID_KEY],
-        fingerprint=kdict[KEY_FINGERPRINT_KEY],
-        key_data=kdict[KEY_DATA_KEY],
-        private=kdict[KEY_PRIVATE_KEY],
-        length=kdict[KEY_LENGTH_KEY],
+        key[KEY_ADDRESS_KEY],
+        key_id=key[KEY_ID_KEY],
+        fingerprint=key[KEY_FINGERPRINT_KEY],
+        key_data=key[KEY_DATA_KEY],
+        private=key[KEY_PRIVATE_KEY],
+        length=key[KEY_LENGTH_KEY],
         expiry_date=expiry_date,
         last_audited_at=last_audited_at,
         refreshed_at=refreshed_at,
         validation=validation,
-        encr_used=kdict[KEY_ENCR_USED_KEY],
-        sign_used=kdict[KEY_SIGN_USED_KEY],
+        encr_used=encr_used,
+        sign_used=sign_used,
     )
 
 
@@ -178,6 +188,7 @@ class EncryptionKey(object):
                  key_data="", private=False, length=0, expiry_date=None,
                  validation=ValidationLevels.Weak_Chain, last_audited_at=None,
                  refreshed_at=None, encr_used=False, sign_used=False):
+        # TODO: it should know its own active address
         self.address = address
         self.key_id = key_id
         self.fingerprint = fingerprint
@@ -185,6 +196,7 @@ class EncryptionKey(object):
         self.private = private
         self.length = length
         self.expiry_date = expiry_date
+
         self.validation = validation
         self.last_audited_at = last_audited_at
         self.refreshed_at = refreshed_at
@@ -199,7 +211,6 @@ class EncryptionKey(object):
         :rtype: str
         """
         expiry_date = _to_unix_time(self.expiry_date)
-        last_audited_at = _to_unix_time(self.last_audited_at)
         refreshed_at = _to_unix_time(self.refreshed_at)
 
         return json.dumps({
@@ -211,11 +222,7 @@ class EncryptionKey(object):
             KEY_PRIVATE_KEY: self.private,
             KEY_LENGTH_KEY: self.length,
             KEY_EXPIRY_DATE_KEY: expiry_date,
-            KEY_LAST_AUDITED_AT_KEY: last_audited_at,
             KEY_REFRESHED_AT_KEY: refreshed_at,
-            KEY_VALIDATION_KEY: str(self.validation),
-            KEY_ENCR_USED_KEY: self.encr_used,
-            KEY_SIGN_USED_KEY: self.sign_used,
             KEY_TAGS_KEY: [KEYMANAGER_KEY_TAG],
         })
 
@@ -223,16 +230,20 @@ class EncryptionKey(object):
         """
         Return a JSON string describing this key.
 
-        :param address: Address for wich the key is active
-        :type address: str
         :return: The JSON string describing this key.
         :rtype: str
         """
+        last_audited_at = _to_unix_time(self.last_audited_at)
+
         return json.dumps({
             KEY_ADDRESS_KEY: address,
             KEY_TYPE_KEY: self.__class__.__name__ + KEYMANAGER_ACTIVE_TYPE,
             KEY_ID_KEY: self.key_id,
             KEY_PRIVATE_KEY: self.private,
+            KEY_VALIDATION_KEY: str(self.validation),
+            KEY_LAST_AUDITED_AT_KEY: last_audited_at,
+            KEY_ENCR_USED_KEY: self.encr_used,
+            KEY_SIGN_USED_KEY: self.sign_used,
             KEY_TAGS_KEY: [KEYMANAGER_ACTIVE_TAG],
         })
 
@@ -464,3 +475,68 @@ class EncryptionScheme(object):
         :rtype: bool
         """
         pass
+
+    def _repair_key_docs(self, doclist):
+        """
+        If there is more than one key for a key id try to self-repair it
+
+        :return: a Deferred that will be fired with the valid key doc once all
+                 the deletions are completed
+        :rtype: Deferred
+        """
+        def log_key_doc(doc):
+            logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
+                                       doc.content[KEY_FINGERPRINT_KEY]))
+
+        def cmp_key(d1, d2):
+            return cmp(d1.content[KEY_REFRESHED_AT_KEY],
+                       d2.content[KEY_REFRESHED_AT_KEY])
+
+        return self._repair_docs(doclist, cmp_key, log_key_doc)
+
+    def _repair_active_docs(self, doclist):
+        """
+        If there is more than one active doc for an address try to self-repair
+        it
+
+        :return: a Deferred that will be fired with the valid active doc once
+                 all the deletions are completed
+        :rtype: Deferred
+        """
+        def log_active_doc(doc):
+            logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
+                                       doc.content[KEY_ID_KEY]))
+
+        def cmp_active(d1, d2):
+            res = cmp(d1.content[KEY_LAST_AUDITED_AT_KEY],
+                      d2.content[KEY_LAST_AUDITED_AT_KEY])
+            if res != 0:
+                return res
+
+            used1 = (d1.content[KEY_SIGN_USED_KEY] +
+                     d1.content[KEY_ENCR_USED_KEY])
+            used2 = (d2.content[KEY_SIGN_USED_KEY] +
+                     d2.content[KEY_ENCR_USED_KEY])
+            return cmp(used1, used2)
+
+        return self._repair_docs(doclist, cmp_active, log_active_doc)
+
+    def _repair_docs(self, doclist, cmp_func, log_func):
+        logger.error("BUG ---------------------------------------------------")
+        logger.error("There is more than one doc of type %s:"
+                     % (doclist[0].content[KEY_TYPE_KEY],))
+
+        doclist.sort(cmp=cmp_func, reverse=True)
+        log_func(doclist[0])
+        deferreds = []
+        for doc in doclist[1:]:
+            log_func(doc)
+            d = self._soledad.delete_doc(doc)
+            deferreds.append(d)
+
+        logger.error("")
+        logger.error(traceback.extract_stack())
+        logger.error("BUG (please report above info) ------------------------")
+        d = defer.gatherResults(deferreds, consumeErrors=True)
+        d.addCallback(lambda _: doclist[0])
+        return d
diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index 9064043..3c8ac1e 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -22,7 +22,6 @@ import os
 import re
 import shutil
 import tempfile
-import traceback
 import io
 
 
@@ -44,8 +43,6 @@ from leap.keymanager.keys import (
     TYPE_ADDRESS_PRIVATE_INDEX,
     KEY_ADDRESS_KEY,
     KEY_ID_KEY,
-    KEY_FINGERPRINT_KEY,
-    KEY_REFRESHED_AT_KEY,
     KEYMANAGER_ACTIVE_TYPE,
 )
 
@@ -223,6 +220,41 @@ class OpenPGPKey(EncryptionKey):
 
         return []
 
+    def merge(self, newkey):
+        if newkey.fingerprint != self.fingerprint:
+            logger.critical(
+                "Can't put a key whith the same key_id and different "
+                "fingerprint: %s, %s"
+                % (newkey.fingerprint, self.fingerprint))
+            raise errors.KeyFingerprintMismatch(newkey.fingerprint)
+
+        with TempGPGWrapper(gpgbinary=self._gpgbinary) as gpg:
+            gpg.import_keys(self.key_data)
+            gpg.import_keys(newkey.key_data)
+            gpgkey = gpg.list_keys(secret=newkey.private).pop()
+
+            if gpgkey['expires']:
+                self.expiry_date = datetime.fromtimestamp(
+                    int(gpgkey['expires']))
+            else:
+                self.expiry_date = None
+
+            self.uids = []
+            for uid in gpgkey['uids']:
+                self.uids.append(_parse_address(uid))
+
+            self.length = int(gpgkey['length'])
+            self.key_data = gpg.export_keys(gpgkey['fingerprint'],
+                                            secret=self.private)
+
+        if newkey.validation > self.validation:
+            self.validation = newkey.validation
+        if newkey.last_audited_at > self.last_audited_at:
+            self.validation = newkey.last_audited_at
+        self.encr_used = newkey.encr_used or self.encr_used
+        self.sign_used = newkey.sign_used or self.sign_used
+        self.refreshed_at = datetime.now()
+
 
 class OpenPGPScheme(EncryptionScheme):
     """
@@ -332,15 +364,16 @@ class OpenPGPScheme(EncryptionScheme):
         """
         address = _parse_address(address)
 
-        def build_key(doc):
-            if doc is None:
+        def build_key((keydoc, activedoc)):
+            if keydoc is None:
                 raise errors.KeyNotFound(address)
             leap_assert(
-                address in doc.content[KEY_ADDRESS_KEY],
+                address in keydoc.content[KEY_ADDRESS_KEY],
                 'Wrong address in key %s. Expected %s, found %s.'
-                % (doc.content[KEY_ID_KEY], address,
-                   doc.content[KEY_ADDRESS_KEY]))
-            key = build_key_from_dict(OpenPGPKey, doc.content)
+                % (keydoc.content[KEY_ID_KEY], address,
+                   keydoc.content[KEY_ADDRESS_KEY]))
+            key = build_key_from_dict(OpenPGPKey, keydoc.content,
+                                      activedoc.content)
             key._gpgbinary = self._gpgbinary
             return key
 
@@ -426,100 +459,44 @@ class OpenPGPScheme(EncryptionScheme):
         :return: A Deferred which fires when the key is in the storage.
         :rtype: Deferred
         """
-        d = self._put_key_doc(key)
-        d.addCallback(lambda _: self._put_active_doc(key, address))
-        return d
+        def merge_and_put((keydoc, activedoc)):
+            if not keydoc:
+                return put_new_key(activedoc)
 
-    def _put_key_doc(self, key):
-        """
-        Put key document in soledad
+            active_content = None
+            if activedoc:
+                active_content = activedoc.content
+            oldkey = build_key_from_dict(OpenPGPKey, keydoc.content,
+                                         active_content)
 
-        :type key: OpenPGPKey
-        :rtype: Deferred
-        """
-        def check_and_put(docs, key):
-            deferred_repair = defer.succeed(None)
-            if len(docs) == 0:
-                return self._soledad.create_doc_from_json(key.get_json())
-            elif len(docs) > 1:
-                deferred_repair = self._repair_key_docs(docs, key.key_id)
-
-            doc = docs[0]
-            oldkey = build_key_from_dict(OpenPGPKey, doc.content)
-            if key.fingerprint != oldkey.fingerprint:
-                logger.critical(
-                    "Can't put a key whith the same key_id and different "
-                    "fingerprint: %s, %s"
-                    % (key.fingerprint, oldkey.fingerprint))
-                return defer.fail(
-                    errors.KeyFingerprintMismatch(key.fingerprint))
-
-            # in case of an update of the key merge them with gnupg
-            with TempGPGWrapper(gpgbinary=self._gpgbinary) as gpg:
-                gpg.import_keys(oldkey.key_data)
-                gpg.import_keys(key.key_data)
-                gpgkey = gpg.list_keys(secret=key.private).pop()
-                mergedkey = self._build_key_from_gpg(
-                    gpgkey,
-                    gpg.export_keys(gpgkey['fingerprint'],
-                                    secret=key.private))
-            mergedkey.validation = max(
-                [key.validation, oldkey.validation])
-            mergedkey.last_audited_at = oldkey.last_audited_at
-            mergedkey.refreshed_at = key.refreshed_at
-            mergedkey.encr_used = key.encr_used or oldkey.encr_used
-            mergedkey.sign_used = key.sign_used or oldkey.sign_used
-            doc.set_json(mergedkey.get_json())
-            deferred_put = self._soledad.put_doc(doc)
-
-            d = defer.gatherResults([deferred_put, deferred_repair])
-            d.addCallback(lambda res: res[0])
-            return d
+            key.merge(oldkey)
+            keydoc.set_json(key.get_json())
+            deferred_key = self._soledad.put_doc(keydoc)
 
-        d = self._soledad.get_from_index(
-            TYPE_ID_PRIVATE_INDEX,
-            self.KEY_TYPE,
-            key.key_id,
-            '1' if key.private else '0')
-        d.addCallback(check_and_put, key)
-        return d
+            active_json = key.get_active_json(address)
+            if activedoc:
+                activedoc.set_json(active_json)
+                deferred_active = self._soledad.put_doc(activedoc)
+            else:
+                deferred_active = self._soledad.create_doc_from_json(
+                    active_json)
 
-    def _put_active_doc(self, key, address):
-        """
-        Put active key document in soledad
+            return defer.gatherResults([deferred_key, deferred_active])
 
-        :type key: OpenPGPKey
-        :type addresses: str
-        :rtype: Deferred
-        """
-        def check_and_put(docs):
-            if len(docs) == 1:
-                doc = docs.pop()
-                doc.set_json(key.get_active_json(address))
-                d = self._soledad.put_doc(doc)
-            else:
-                if len(docs) > 1:
-                    logger.error("There is more than one active key document "
-                                 "for the address %s" % (address,))
-                    deferreds = []
-                    for doc in docs:
-                        delete = self._soledad.delete_doc(doc)
-                        deferreds.append(delete)
-                    d = defer.gatherResults(deferreds, consumeErrors=True)
-                else:
-                    d = defer.succeed(None)
-
-                d.addCallback(
-                    lambda _: self._soledad.create_doc_from_json(
-                        key.get_active_json(address)))
-            return d
+        def put_new_key(activedoc):
+            deferreds = []
+            if activedoc:
+                d = self._soledad.delete_doc(activedoc)
+                deferreds.append(d)
+            for json in [key.get_json(), key.get_active_json(address)]:
+                d = self._soledad.create_doc_from_json(json)
+                deferreds.append(d)
+            return defer.gatherResults(deferreds)
 
-        d = self._soledad.get_from_index(
-            TYPE_ADDRESS_PRIVATE_INDEX,
-            self.ACTIVE_TYPE,
-            address,
-            '1' if key.private else '0')
-        d.addCallback(check_and_put)
+        dk = self._get_key_doc_from_keyid(key.key_id, key.private)
+        da = self._get_active_doc_from_address(address, key.private)
+        d = defer.gatherResults([dk, da])
+        d.addCallback(merge_and_put)
         return d
 
     def _get_key_doc(self, address, private=False):
@@ -533,44 +510,26 @@ class OpenPGPScheme(EncryptionScheme):
         :param private: Whether to look for a private key.
         :type private: bool
 
-        :return: A Deferred which fires with the SoledadDocument with the key
-                 or None if it does not exist.
+        :return: A Deferred which fires with a touple of two SoledadDocument
+                 (keydoc, activedoc) or None if it does not exist.
         :rtype: Deferred
         """
         def get_key_from_active_doc(activedoc):
-            if len(activedoc) is 0:
-                return None
-            leap_assert(
-                len(activedoc) is 1,
-                'Found more than one key for address %s!' % (address,))
-
-            key_id = activedoc[0].content[KEY_ID_KEY]
-            d = self._soledad.get_from_index(
-                TYPE_ID_PRIVATE_INDEX,
-                self.KEY_TYPE,
-                key_id,
-                '1' if private else '0')
-            d.addCallback(get_doc, key_id, activedoc)
+            if not activedoc:
+                return (None, None)
+            key_id = activedoc.content[KEY_ID_KEY]
+            d = self._get_key_doc_from_keyid(key_id, private)
+            d.addCallback(delete_active_if_no_key, activedoc)
             return d
 
-        def get_doc(doclist, key_id, activedoc):
-            if len(doclist) == 0:
-                logger.warning('There is no key for id %s! Self-repairing it.'
-                               % (key_id))
+        def delete_active_if_no_key(keydoc, activedoc):
+            if not keydoc:
                 d = self._soledad.delete_doc(activedoc)
-                d.addCallback(lambda _: None)
-                return d
-            elif len(doclist) > 1:
-                d = self._repair_key_docs(doclist, key_id)
-                d.addCallback(lambda _: doclist[0])
+                d.addCallback(lambda _: (None, None))
                 return d
-            return doclist[0]
+            return (keydoc, activedoc)
 
-        d = self._soledad.get_from_index(
-            TYPE_ADDRESS_PRIVATE_INDEX,
-            self.ACTIVE_TYPE,
-            address,
-            '1' if private else '0')
+        d = self._get_active_doc_from_address(address, private)
         d.addCallback(get_key_from_active_doc)
         return d
 
@@ -647,36 +606,6 @@ class OpenPGPScheme(EncryptionScheme):
         d.addCallback(delete_key)
         return d
 
-    def _repair_key_docs(self, doclist, key_id):
-        """
-        If there is more than one key for a key id try to self-repair it
-
-        :return: a Deferred that will be fired once all the deletions are
-                 completed
-        :rtype: Deferred
-        """
-        logger.error("BUG ---------------------------------------------------")
-        logger.error("There is more than one key with the same key_id %s:"
-                     % (key_id,))
-
-        def log_key_doc(doc):
-            logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
-                                       doc.content[KEY_FINGERPRINT_KEY]))
-
-        doclist.sort(key=lambda doc: doc.content[KEY_REFRESHED_AT_KEY],
-                     reverse=True)
-        log_key_doc(doclist[0])
-        deferreds = []
-        for doc in doclist[1:]:
-            log_key_doc(doc)
-            d = self._soledad.delete_doc(doc)
-            deferreds.append(d)
-
-        logger.error("")
-        logger.error(traceback.extract_stack())
-        logger.error("BUG (please report above info) ------------------------")
-        return defer.gatherResults(deferreds, consumeErrors=True)
-
     #
     # Data encryption, decryption, signing and verifying
     #
@@ -885,6 +814,31 @@ class OpenPGPScheme(EncryptionScheme):
             kfprint = gpgpubkey['fingerprint']
             return valid and rfprint == kfprint
 
+    def _get_active_doc_from_address(self, address, private):
+        d = self._soledad.get_from_index(
+            TYPE_ADDRESS_PRIVATE_INDEX,
+            self.ACTIVE_TYPE,
+            address,
+            '1' if private else '0')
+        d.addCallback(self._repair_and_get_doc, self._repair_active_docs)
+        return d
+
+    def _get_key_doc_from_keyid(self, key_id, private):
+        d = self._soledad.get_from_index(
+            TYPE_ID_PRIVATE_INDEX,
+            self.KEY_TYPE,
+            key_id,
+            '1' if private else '0')
+        d.addCallback(self._repair_and_get_doc, self._repair_key_docs)
+        return d
+
+    def _repair_and_get_doc(self, doclist, repair_func):
+        if len(doclist) is 0:
+            return None
+        elif len(doclist) > 1:
+            return repair_func(doclist)
+        return doclist[0]
+
 
 def process_ascii_key(key_data, gpgbinary, secret=False):
     with TempGPGWrapper(gpgbinary=gpgbinary) as gpg:
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index 0f08326..e4e0d8b 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -83,13 +83,18 @@ class KeyManagerUtilTestCase(unittest.TestCase):
             'private': False,
             'length': 4096,
             'expiry_date': 0,
-            'last_audited_at': 0,
             'refreshed_at': 1311239602,
+        }
+        adict = {
+            'address': ADDRESS,
+            'key_id': KEY_FINGERPRINT[-16:],
+            'private': False,
+            'last_audited_at': 0,
             'validation': str(ValidationLevels.Weak_Chain),
             'encr_used': False,
             'sign_used': True,
         }
-        key = build_key_from_dict(OpenPGPKey, kdict)
+        key = build_key_from_dict(OpenPGPKey, kdict, adict)
         self.assertEqual(
             kdict['address'], key.address,
             'Wrong data in key.')
@@ -118,13 +123,13 @@ class KeyManagerUtilTestCase(unittest.TestCase):
             datetime.fromtimestamp(kdict['refreshed_at']), key.refreshed_at,
             'Wrong data in key.')
         self.assertEqual(
-            ValidationLevels.get(kdict['validation']), key.validation,
+            ValidationLevels.get(adict['validation']), key.validation,
             'Wrong data in key.')
         self.assertEqual(
-            kdict['encr_used'], key.encr_used,
+            adict['encr_used'], key.encr_used,
             'Wrong data in key.')
         self.assertEqual(
-            kdict['sign_used'], key.sign_used,
+            adict['sign_used'], key.sign_used,
             'Wrong data in key.')
 
 
diff --git a/src/leap/keymanager/tests/test_openpgp.py b/src/leap/keymanager/tests/test_openpgp.py
index 96b40a0..6641591 100644
--- a/src/leap/keymanager/tests/test_openpgp.py
+++ b/src/leap/keymanager/tests/test_openpgp.py
@@ -29,7 +29,10 @@ from leap.keymanager import (
     KeyNotFound,
     openpgp,
 )
-from leap.keymanager.keys import TYPE_ID_PRIVATE_INDEX
+from leap.keymanager.keys import (
+    TYPE_ID_PRIVATE_INDEX,
+    TYPE_ADDRESS_PRIVATE_INDEX,
+)
 from leap.keymanager.openpgp import OpenPGPKey
 from leap.keymanager.tests import (
     KeyManagerWithSoledadTestCase,
@@ -308,6 +311,7 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         try:
             yield self.assertFailure(pgp.get_key(ADDRESS, private=False),
                                      KeyNotFound)
+            # it should have deleted the index
             self.assertEqual(self._soledad.delete_doc.call_count, 1)
         finally:
             self._soledad.get_from_index = get_from_index
@@ -349,6 +353,54 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
             self._soledad.get_from_index = get_from_index
             self._soledad.delete_doc = delete_doc
 
+    @inlineCallbacks
+    def test_self_repair_five_active_docs(self):
+        pgp = openpgp.OpenPGPScheme(
+            self._soledad, gpgbinary=self.gpg_binary_path)
+
+        get_from_index = self._soledad.get_from_index
+        delete_doc = self._soledad.delete_doc
+
+        def my_get_from_index(*args):
+            if (args[0] == TYPE_ADDRESS_PRIVATE_INDEX and
+                    args[2] == ADDRESS):
+                k1 = OpenPGPKey(ADDRESS, key_id="1",
+                                last_audited_at=datetime(2005, 1, 1))
+                k2 = OpenPGPKey(ADDRESS, key_id="2",
+                                last_audited_at=datetime(2007, 1, 1))
+                k3 = OpenPGPKey(ADDRESS, key_id="3",
+                                last_audited_at=datetime(2007, 1, 1),
+                                encr_used=True, sign_used=True)
+                k4 = OpenPGPKey(ADDRESS, key_id="4",
+                                last_audited_at=datetime(2007, 1, 1),
+                                sign_used=True)
+                k5 = OpenPGPKey(ADDRESS, key_id="5",
+                                last_audited_at=datetime(2007, 1, 1),
+                                encr_used=True)
+                deferreds = []
+                for k in [k1, k2, k3, k4, k5]:
+                    d = self._soledad.create_doc_from_json(
+                        k.get_active_json(ADDRESS))
+                    deferreds.append(d)
+                return gatherResults(deferreds)
+            elif args[0] == TYPE_ID_PRIVATE_INDEX:
+                key_id = args[2]
+                self.assertEqual(key_id, "3")
+                k = OpenPGPKey(ADDRESS, key_id="3")
+                return succeed(
+                    [self._soledad.create_doc_from_json(k.get_json())])
+            return get_from_index(*args)
+
+        self._soledad.get_from_index = my_get_from_index
+        self._soledad.delete_doc = Mock(return_value=succeed(None))
+
+        try:
+            yield pgp.get_key(ADDRESS, private=False)
+            self.assertEqual(self._soledad.delete_doc.call_count, 4)
+        finally:
+            self._soledad.get_from_index = get_from_index
+            self._soledad.delete_doc = delete_doc
+
     def _assert_key_not_found(self, pgp, address, private=False):
         d = pgp.get_key(address, private=private)
         return self.assertFailure(d, KeyNotFound)
diff --git a/src/leap/keymanager/tests/test_validation.py b/src/leap/keymanager/tests/test_validation.py
index bcf41c4..44d6e39 100644
--- a/src/leap/keymanager/tests/test_validation.py
+++ b/src/leap/keymanager/tests/test_validation.py
@@ -108,14 +108,14 @@ class ValidationLevelsTestCase(KeyManagerWithSoledadTestCase):
         TEXT = "some text"
 
         km = self._key_manager()
+        yield km.put_raw_key(UNEXPIRED_PRIVATE, OpenPGPKey, ADDRESS)
+        signature = yield km.sign(TEXT, ADDRESS, OpenPGPKey)
+        yield self.delete_all_keys(km)
+
         yield km.put_raw_key(UNEXPIRED_KEY, OpenPGPKey, ADDRESS)
         yield km.encrypt(TEXT, ADDRESS, OpenPGPKey)
-
-        km2 = self._key_manager()
-        yield km2.put_raw_key(UNEXPIRED_PRIVATE, OpenPGPKey, ADDRESS)
-        signature = yield km2.sign(TEXT, ADDRESS, OpenPGPKey)
-
         yield km.verify(TEXT, ADDRESS, OpenPGPKey, detached_sig=signature)
+
         d = km.put_raw_key(
             UNRELATED_KEY, OpenPGPKey, ADDRESS,
             validation=ValidationLevels.Provider_Endorsement)
@@ -126,17 +126,17 @@ class ValidationLevelsTestCase(KeyManagerWithSoledadTestCase):
         TEXT = "some text"
 
         km = self._key_manager()
+        yield km.put_raw_key(UNEXPIRED_PRIVATE, OpenPGPKey, ADDRESS)
+        yield km.put_raw_key(PUBLIC_KEY_2, OpenPGPKey, ADDRESS_2)
+        encrypted = yield km.encrypt(TEXT, ADDRESS_2, OpenPGPKey,
+                                     sign=ADDRESS)
+        yield self.delete_all_keys(km)
+
         yield km.put_raw_key(UNEXPIRED_KEY, OpenPGPKey, ADDRESS)
         yield km.put_raw_key(PRIVATE_KEY_2, OpenPGPKey, ADDRESS_2)
         yield km.encrypt(TEXT, ADDRESS, OpenPGPKey)
-
-        km2 = self._key_manager()
-        yield km2.put_raw_key(UNEXPIRED_PRIVATE, OpenPGPKey, ADDRESS)
-        yield km2.put_raw_key(PUBLIC_KEY_2, OpenPGPKey, ADDRESS_2)
-        encrypted = yield km2.encrypt(TEXT, ADDRESS_2, OpenPGPKey,
-                                      sign=ADDRESS)
-
         yield km.decrypt(encrypted, ADDRESS_2, OpenPGPKey, verify=ADDRESS)
+
         d = km.put_raw_key(
             UNRELATED_KEY, OpenPGPKey, ADDRESS,
             validation=ValidationLevels.Provider_Endorsement)
@@ -150,6 +150,33 @@ class ValidationLevelsTestCase(KeyManagerWithSoledadTestCase):
         key = yield km.get_key(ADDRESS, OpenPGPKey, fetch_remote=False)
         self.assertEqual(key.fingerprint, SIGNED_FINGERPRINT)
 
+    @inlineCallbacks
+    def test_two_uuids(self):
+        TEXT = "some text"
+
+        km = self._key_manager()
+        yield km.put_raw_key(UUIDS_PRIVATE, OpenPGPKey, ADDRESS_2)
+        signature = yield km.sign(TEXT, ADDRESS_2, OpenPGPKey)
+        yield self.delete_all_keys(km)
+
+        yield km.put_raw_key(UUIDS_KEY, OpenPGPKey, ADDRESS_2)
+        yield km.put_raw_key(UUIDS_KEY, OpenPGPKey, ADDRESS)
+        yield km.encrypt(TEXT, ADDRESS_2, OpenPGPKey)
+        yield km.verify(TEXT, ADDRESS_2, OpenPGPKey, detached_sig=signature)
+
+        d = km.put_raw_key(
+            PUBLIC_KEY_2, OpenPGPKey, ADDRESS_2,
+            validation=ValidationLevels.Provider_Endorsement)
+        yield self.assertFailure(d, KeyNotValidUpgrade)
+        key = yield km.get_key(ADDRESS_2, OpenPGPKey, fetch_remote=False)
+        self.assertEqual(key.fingerprint, UUIDS_FINGERPRINT)
+
+        yield km.put_raw_key(
+            PUBLIC_KEY, OpenPGPKey, ADDRESS,
+            validation=ValidationLevels.Provider_Endorsement)
+        key = yield km.get_key(ADDRESS, OpenPGPKey, fetch_remote=False)
+        self.assertEqual(key.fingerprint, KEY_FINGERPRINT)
+
 
 # Key material for testing
 
@@ -364,6 +391,117 @@ X2+l7IOSt+31KQCBFN/VmhTySJOVQC1d2A56lSH2c/DWVClji+x3suzn
 -----END PGP PUBLIC KEY BLOCK-----
 """
 
+# key 0x1DDBAEB928D982F7: public key two uuids
+#    uid                  anotheruser <anotheruser@leap.se>
+#    uid                  Leap Test Key <leap@leap.se>
+UUIDS_FINGERPRINT = "21690ED054C1B2F3ACE963D38FCC7DEFB4EE5A9B"
+UUIDS_KEY = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQENBFZwjz0BCADHpVg7js8PK9gQXx3Z+Jtj6gswYZpeXRdIUfZBSebWNGKwXxC9
+ZZDjnQc3l6Kezh7ra/hB6xulDbj3vXi66V279QSOuFAKMIudlJehb86bUiVk9Ppy
+kdrn44P40ZdVmw2m61WrKnvBelKW7pIF/EB/dY1uUonSfR56f/BxL5a5jGi2uI2y
+2hTPnZEksoKQsjsp1FckPFwVGzRKVzYl3lsorL5oiHd450G2B3VRw8AZ8Eqq6bzf
+jNrrA3TOMmEIYdADPqqnBj+xbnOCGBsvx+iAhGRxUckVJqW92SXlNNds8kEyoE0t
+9B6eqe0MrrlcK3SLe03j85T9f1o3An53rV/3ABEBAAG0HExlYXAgVGVzdCBLZXkg
+PGxlYXBAbGVhcC5zZT6JAT0EEwEIACcFAlZwjz0CGwMFCRLMAwAFCwkIBwMFFQoJ
+CAsFFgMCAQACHgECF4AACgkQj8x977TuWpu4ZggAgk6rVtOCqYwM720Bs3k+wsWu
+IVPUqnlPbSWph/PKBKWYE/5HoIGdvfN9jJxwpCM5x/ivPe1zeJ0qa9SnO66kolHU
+7qC8HRWC67R4znO4Zrs2I+SgwRHAPPbqMVPsNs5rS0D6DCcr+LXtJF+LLAsIwDfw
+mXEsKbX5H5aBmmDnfq0pGx05E3tKs5l09VVESvVZYOCM9b4FtdLVvgbKAD+KYDW1
+5A/PzOvyYjZu2FGtPKmNmqHD3KW8cmrcI/7mRR08FnNGbbpgXPZ2GPKgkUllY9N7
+E9lg4eaYH2fIWun293kbqp8ueELZvoU1jUQrP5B+eqBWTvIucqdQqJaiWn9pELQh
+YW5vdGhlcnVzZXIgPGFub3RoZXJ1c2VyQGxlYXAuc2U+iQE9BBMBCAAnBQJWcI9a
+AhsDBQkSzAMABQsJCAcDBRUKCQgLBRYDAgEAAh4BAheAAAoJEI/Mfe+07lqblRMH
+/17vK2WOd0F7EegA5ELOrM+QJPKpLK4e6WdROUpS1hvRQcAOqadCCpYPSTTG/HnO
+d9/Q9Q/G3xHHlk6nl1qHRkVlp0iVWyBZFn1s8lgGz/FFUEXXRj7I5RGmKSNgDlqA
+un2OrwB2m+DH6pMjizu/RUfIJM2bSgViuvjCQoaLYLcFiULJlWHb/2WgpvesFyAc
+0oc9AkXuaCEo50XQlrt8Bh++6rfbAMAS7ZrHluFPIY+y4eVl+MU/QkoGYAVgiLLV
+5tnvbDZWNs8ixw4ubqKXYj5mK55sapokhOqObEfY6D3p7YpdQO/IhBneCw9gKOxa
+wYAPhCOrJC8JmE69I1Nk8Bu5AQ0EVnCPPQEIANUivsrR2jwb8C9wPONn0HS3YYCI
+/IVlLdw/Ia23ogBF1Uh8ORNg1G/t0/6S7IKDZ2gGgWw25u9TjWRRWsxO9tjOPi2d
+YuhwlQRnq5Or+LzIEKRs9GnJMLFT0kR9Nrhw4UyaN6tWkR9p1Py7ux8RLmDEMAs3
+fBfVMLMmQRerJ5SyCUiq/lm9aYTLCC+vkjE01C+2oI0gcWGfLDjiJbaD4AazzibP
+fBe41BIk7WaCJyEcBqfrgW/Seof43FhSKRGgc5nx3HH1DMz9AtYfKnVS5DgoBGpO
+hxgzIJN3/hFHPicRlYoHxLUE48GcFfQFEJ78RXeBuieXAkDxNczHnLkEPx8AEQEA
+AYkBJQQYAQgADwUCVnCPPQIbDAUJEswDAAAKCRCPzH3vtO5amyRIB/9IsWUWrvbH
+njw2ZCiIV++lHgIntAcuQIbZIjyMXoM+imHsPrsDOUT65yi9Xp1pUyZEKtGkZvP4
+p7HRzJL+RWiWEN7sgQfNqqev8BF2/xmxkmiSuXHJ0PSaC5DmLfFDyvvSU6H5VPud
+NszKIHtyoS6ph6TH8GXzFmNiHaTOZKdmVxlyLj1/DN+d63M+ARZIe7gB6jmP/gg4
+o52icfTcqLkkppYn8g1A9bdJ3k8qqExNPTf2llDscuD9VzpebFbPqfcYqR71GfG7
+Kmg7qGnZtNac1ERvknI/fmmCQydGk5pOh0KUTgeLG2qB07cqCUBbOXaweNWbiM9E
+vtQLNMD9Gn7D
+=MCXv
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+UUIDS_PRIVATE = """
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1
+
+lQOYBFZwjz0BCADHpVg7js8PK9gQXx3Z+Jtj6gswYZpeXRdIUfZBSebWNGKwXxC9
+ZZDjnQc3l6Kezh7ra/hB6xulDbj3vXi66V279QSOuFAKMIudlJehb86bUiVk9Ppy
+kdrn44P40ZdVmw2m61WrKnvBelKW7pIF/EB/dY1uUonSfR56f/BxL5a5jGi2uI2y
+2hTPnZEksoKQsjsp1FckPFwVGzRKVzYl3lsorL5oiHd450G2B3VRw8AZ8Eqq6bzf
+jNrrA3TOMmEIYdADPqqnBj+xbnOCGBsvx+iAhGRxUckVJqW92SXlNNds8kEyoE0t
+9B6eqe0MrrlcK3SLe03j85T9f1o3An53rV/3ABEBAAEAB/9Lzeg2lP7hz8/2R2da
+QB8gTNl6wVSPx+DzQMuz9o+DfdiLB02f3FSrWBBJd3XzvmfXE+Prg423mgJFbtfM
+gJdqqpnUZv9dHxmj96urTHyyVPqF3s7JecAYlDaj31EK3BjO7ERW/YaH7B432NXx
+F9qVitjsrsJN/dv4v2NYVq1wPcdDB05ge9WP+KRec7xvdTudH4Kov0iMZ+1Nksfn
+lrowGuMYBGWDlTNoBoEYxDD2lqVaiOjyjx5Ss8btS59IlXxApOFZTezekl7hUI2B
+1fDQ1GELL6BKVKxApGSD5XAgVlqkek4RhoHmg4gKSymfbFV5oRp1v1kC0JIGvnB1
+W5/BBADKzagL4JRnhWGubLec917B3se/3y1aHrEmO3T0IzxnUMD5yvg81uJWi5Co
+M05Nu/Ny/Fw1VgoF8MjiGnumB2KKytylu8LKLarDxPpLxabOBCQLHrLQOMsmesjR
+Cg3iYl/EeM/ooAufaN4IObcu6Pa8//rwNE7Fz1ZsIyJefN4fnwQA/AOpqA2BvHoH
+VRYh4NVuMLhF1YdKqcd/T/dtSqszcadkmG4+vAL76r3jYxScRoNGQaIkpBnzP0ry
+Adb0NDuBgSe/Cn44kqT7JJxTMfJNrw2rBMMUYZHdQrck2pf5R4fZ74yJyvCKg5pQ
+QAl1gTSi6PJvPwpc7m7Kr4kHBVDlgKkEAJKkVrtq/v2+jSA+/osa4YC5brsDQ08X
+pvZf0MBkc5/GDfusHyE8HGFnVY5ycrS85TBCrhc7suFu59pF4wEeXsqxqNf02gRe
+B+btPwR7yki73iyXs4cbuszHMD03UnbvItFAybD5CC+oR9kG5noI0TzJNUNX9Vkq
+xATf819dhwtgTha0HExlYXAgVGVzdCBLZXkgPGxlYXBAbGVhcC5zZT6JAT0EEwEI
+ACcFAlZwjz0CGwMFCRLMAwAFCwkIBwMFFQoJCAsFFgMCAQACHgECF4AACgkQj8x9
+77TuWpu4ZggAgk6rVtOCqYwM720Bs3k+wsWuIVPUqnlPbSWph/PKBKWYE/5HoIGd
+vfN9jJxwpCM5x/ivPe1zeJ0qa9SnO66kolHU7qC8HRWC67R4znO4Zrs2I+SgwRHA
+PPbqMVPsNs5rS0D6DCcr+LXtJF+LLAsIwDfwmXEsKbX5H5aBmmDnfq0pGx05E3tK
+s5l09VVESvVZYOCM9b4FtdLVvgbKAD+KYDW15A/PzOvyYjZu2FGtPKmNmqHD3KW8
+cmrcI/7mRR08FnNGbbpgXPZ2GPKgkUllY9N7E9lg4eaYH2fIWun293kbqp8ueELZ
+voU1jUQrP5B+eqBWTvIucqdQqJaiWn9pELQhYW5vdGhlcnVzZXIgPGFub3RoZXJ1
+c2VyQGxlYXAuc2U+iQE9BBMBCAAnBQJWcI9aAhsDBQkSzAMABQsJCAcDBRUKCQgL
+BRYDAgEAAh4BAheAAAoJEI/Mfe+07lqblRMH/17vK2WOd0F7EegA5ELOrM+QJPKp
+LK4e6WdROUpS1hvRQcAOqadCCpYPSTTG/HnOd9/Q9Q/G3xHHlk6nl1qHRkVlp0iV
+WyBZFn1s8lgGz/FFUEXXRj7I5RGmKSNgDlqAun2OrwB2m+DH6pMjizu/RUfIJM2b
+SgViuvjCQoaLYLcFiULJlWHb/2WgpvesFyAc0oc9AkXuaCEo50XQlrt8Bh++6rfb
+AMAS7ZrHluFPIY+y4eVl+MU/QkoGYAVgiLLV5tnvbDZWNs8ixw4ubqKXYj5mK55s
+apokhOqObEfY6D3p7YpdQO/IhBneCw9gKOxawYAPhCOrJC8JmE69I1Nk8BudA5gE
+VnCPPQEIANUivsrR2jwb8C9wPONn0HS3YYCI/IVlLdw/Ia23ogBF1Uh8ORNg1G/t
+0/6S7IKDZ2gGgWw25u9TjWRRWsxO9tjOPi2dYuhwlQRnq5Or+LzIEKRs9GnJMLFT
+0kR9Nrhw4UyaN6tWkR9p1Py7ux8RLmDEMAs3fBfVMLMmQRerJ5SyCUiq/lm9aYTL
+CC+vkjE01C+2oI0gcWGfLDjiJbaD4AazzibPfBe41BIk7WaCJyEcBqfrgW/Seof4
+3FhSKRGgc5nx3HH1DMz9AtYfKnVS5DgoBGpOhxgzIJN3/hFHPicRlYoHxLUE48Gc
+FfQFEJ78RXeBuieXAkDxNczHnLkEPx8AEQEAAQAH+wRSCn0RCPP7+v/zLgDMG3Eq
+QHs7C6dmmCnlS7j6Rnnr8HliL0QBy/yi3Q/Fia7RnBiDPT9k04SZdH3KmmUW2rEl
+aSRCkv00PwkSUuuQ6l9lTNUQclnsnqSRlusVgLT3cNG9NJCwFgwFeLBQ2+ey0PZc
+M78edlEDXNPc3CfvK8O7WK74YiNJqIQCs7aDJSv0s7O/asRQyMCsl/UYtMV6W03d
+eauS3bM41ll7GVfHMgkChFUQHb+19JHzSq4yeqQ/vS30ASugFxB3Omomp95sRL/w
+60y51faLyTKD4AN3FhDfeIEfh1ggN2UT70qzC3+F8TvxQQHEBhNQKlhWVbWTp+0E
+ANkcyokvn+09OIO/YDxF3aB37gA3J37d6NXos6pdvqUPOVSHvmF/hiM0FO2To6vu
+ex/WcDQafPm4eiW6oNllwtZhWU2tr34bZD4PIuktSX2Ax2v5QtZ4d1CVdDEwbYn/
+fmR+nif1fTKTljZragaI9Rt6NWhfh7UGt62iIKh0lbhLBAD7T5wHY8V1yqlnyByG
+K7nt+IHnND2I7Hk58yxKjv2KUNYxWZeOAQTQmfQXjJ+BOmw6oHMmDmGvdjSxIE+9
+j9nezEONxzVVjEDTKBeEnUeDY1QGDyDyW1/AhLJ52yWGTNrmKcGV4KmaYnhDzq7Z
+aqJVRcFMF9TAfhrEGGhRdD83/QQA6xAqjWiu6tbaDurVuce64mA1R3T7HJ81gEaX
+I+eJNDJb7PK3dhFETgyc3mcepWFNJkoXqx2ADhG8jLqK4o/N/QlV5PQgeHmhz09V
+Z7MNhivGxDKZoxX6Bouh+qs5OkatcGFhTz//+FHSfusV2emxNiwd4QIsizxaltqh
+W1ke0bA7eYkBJQQYAQgADwUCVnCPPQIbDAUJEswDAAAKCRCPzH3vtO5amyRIB/9I
+sWUWrvbHnjw2ZCiIV++lHgIntAcuQIbZIjyMXoM+imHsPrsDOUT65yi9Xp1pUyZE
+KtGkZvP4p7HRzJL+RWiWEN7sgQfNqqev8BF2/xmxkmiSuXHJ0PSaC5DmLfFDyvvS
+U6H5VPudNszKIHtyoS6ph6TH8GXzFmNiHaTOZKdmVxlyLj1/DN+d63M+ARZIe7gB
+6jmP/gg4o52icfTcqLkkppYn8g1A9bdJ3k8qqExNPTf2llDscuD9VzpebFbPqfcY
+qR71GfG7Kmg7qGnZtNac1ERvknI/fmmCQydGk5pOh0KUTgeLG2qB07cqCUBbOXaw
+eNWbiM9EvtQLNMD9Gn7D
+=/3u/
+-----END PGP PRIVATE KEY BLOCK-----
+"""
+
 if __name__ == "__main__":
-    import unittest
     unittest.main()
-- 
cgit v1.2.3


From 81232da09286f7f1812f6d3d182cd57665feaa1f Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Fri, 18 Dec 2015 19:37:44 +0100
Subject: [feat] Migrate soledad documents by adding versioning field

- Resolves: #7713
---
 src/leap/keymanager/keys.py           |  22 ++++-
 src/leap/keymanager/migrator.py       | 171 ++++++++++++++++++++++++++++++++++
 src/leap/keymanager/tests/__init__.py |   2 +-
 3 files changed, 190 insertions(+), 5 deletions(-)
 create mode 100644 src/leap/keymanager/migrator.py

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index 0de6a82..a60c19d 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -45,6 +45,7 @@ logger = logging.getLogger(__name__)
 # Dictionary keys used for storing cryptographic keys.
 #
 
+KEY_VERSION_KEY = 'version'
 KEY_ADDRESS_KEY = 'address'
 KEY_TYPE_KEY = 'type'
 KEY_ID_KEY = 'key_id'
@@ -69,6 +70,10 @@ KEYMANAGER_KEY_TAG = 'keymanager-key'
 KEYMANAGER_ACTIVE_TAG = 'keymanager-active'
 KEYMANAGER_ACTIVE_TYPE = '-active'
 
+# Version of the Soledad Document schema,
+# it should be bumped each time the document format changes
+KEYMANAGER_DOC_VERSION = 1
+
 
 #
 # key indexing constants.
@@ -223,6 +228,7 @@ class EncryptionKey(object):
             KEY_LENGTH_KEY: self.length,
             KEY_EXPIRY_DATE_KEY: expiry_date,
             KEY_REFRESHED_AT_KEY: refreshed_at,
+            KEY_VERSION_KEY: KEYMANAGER_DOC_VERSION,
             KEY_TAGS_KEY: [KEYMANAGER_KEY_TAG],
         })
 
@@ -244,6 +250,7 @@ class EncryptionKey(object):
             KEY_LAST_AUDITED_AT_KEY: last_audited_at,
             KEY_ENCR_USED_KEY: self.encr_used,
             KEY_SIGN_USED_KEY: self.sign_used,
+            KEY_VERSION_KEY: KEYMANAGER_DOC_VERSION,
             KEY_TAGS_KEY: [KEYMANAGER_ACTIVE_TAG],
         })
 
@@ -281,7 +288,8 @@ class EncryptionScheme(object):
         :type soledad: leap.soledad.Soledad
         """
         self._soledad = soledad
-        self._init_indexes()
+        self.deferred_init = self._init_indexes()
+        self.deferred_init.addCallback(self._migrate_documents_schema)
 
     def _init_indexes(self):
         """
@@ -309,8 +317,14 @@ class EncryptionScheme(object):
                     deferreds.append(d)
             return defer.gatherResults(deferreds, consumeErrors=True)
 
-        self.deferred_indexes = self._soledad.list_indexes()
-        self.deferred_indexes.addCallback(init_idexes)
+        d = self._soledad.list_indexes()
+        d.addCallback(init_idexes)
+        return d
+
+    def _migrate_documents_schema(self, _):
+        from leap.keymanager.migrator import KeyDocumentsMigrator
+        migrator = KeyDocumentsMigrator(self._soledad)
+        return migrator.migrate()
 
     def _wait_indexes(self, *methods):
         """
@@ -343,7 +357,7 @@ class EncryptionScheme(object):
             self.stored[method] = getattr(self, method)
             setattr(self, method, makeWrapper(method))
 
-        self.deferred_indexes.addCallback(restore)
+        self.deferred_init.addCallback(restore)
 
     @abstractmethod
     def get_key(self, address, private=False):
diff --git a/src/leap/keymanager/migrator.py b/src/leap/keymanager/migrator.py
new file mode 100644
index 0000000..b59647a
--- /dev/null
+++ b/src/leap/keymanager/migrator.py
@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+# migrator.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Document migrator
+"""
+# XXX: versioning has being added 12/2015 when keymanager was not
+#      much in use in the wild. We can probably drop support for
+#      keys without version at some point.
+
+
+from collections import namedtuple
+from twisted.internet.defer import gatherResults, succeed
+
+from leap.keymanager.keys import (
+    TAGS_PRIVATE_INDEX,
+    KEYMANAGER_KEY_TAG,
+    KEYMANAGER_ACTIVE_TAG,
+
+    KEYMANAGER_DOC_VERSION,
+    KEY_VERSION_KEY,
+    KEY_ID_KEY,
+    KEY_VALIDATION_KEY,
+    KEY_LAST_AUDITED_AT_KEY,
+    KEY_ENCR_USED_KEY,
+    KEY_SIGN_USED_KEY,
+)
+from leap.keymanager.validation import ValidationLevels
+
+
+KeyDocs = namedtuple("KeyDocs", ['key', 'active'])
+
+
+class KeyDocumentsMigrator(object):
+    """
+    Migrate old KeyManager Soledad Documents to the newest schema
+    """
+
+    def __init__(self, soledad):
+        self._soledad = soledad
+
+    def migrate(self):
+        deferred_public = self._get_docs(private=False)
+        deferred_public.addCallback(self._migrate_docs)
+
+        deferred_private = self._get_docs(private=True)
+        deferred_private.addCallback(self._migrate_docs)
+
+        return gatherResults([deferred_public, deferred_private])
+
+    def _get_docs(self, private=False):
+        private_value = '1' if private else '0'
+
+        deferred_keys = self._soledad.get_from_index(
+            TAGS_PRIVATE_INDEX,
+            KEYMANAGER_KEY_TAG,
+            private_value)
+        deferred_active = self._soledad.get_from_index(
+            TAGS_PRIVATE_INDEX,
+            KEYMANAGER_ACTIVE_TAG,
+            private_value)
+        return gatherResults([deferred_keys, deferred_active])
+
+    def _migrate_docs(self, (key_docs, active_docs)):
+        def update_keys(keys):
+            deferreds = []
+            for key_id in keys:
+                key = keys[key_id].key
+                actives = keys[key_id].active
+
+                d = self._migrate_actives(key, actives)
+                deferreds.append(d)
+
+                d = self._migrate_key(key)
+                deferreds.append(d)
+            return gatherResults(deferreds)
+
+        d = self._buildKeyDict(key_docs, active_docs)
+        d.addCallback(lambda keydict: self._filter_outdated(keydict))
+        d.addCallback(update_keys)
+
+    def _buildKeyDict(self, keys, actives):
+        keydict = {
+            fp2id(key.content[KEY_FINGERPRINT_KEY]): KeyDocs(key, [])
+            for key in keys}
+
+        deferreds = []
+        for active in actives:
+            if KEY_ID_KEY in active.content:
+                key_id = active.content[KEY_ID_KEY]
+                if key_id not in keydict:
+                    d = self._soledad.delete_doc(active)
+                    deferreds.append(d)
+                    continue
+                keydict[key_id].active.append(active)
+
+        d = gatherResults(deferreds)
+        d.addCallback(lambda _: keydict)
+        return d
+
+    def _filter_outdated(self, keydict):
+        outdated = {}
+        for key_id, docs in keydict.items():
+            if ((docs.key and KEY_VERSION_KEY not in docs.key.content) or
+                    docs.active):
+                outdated[key_id] = docs
+        return outdated
+
+    def _migrate_actives(self, key, actives):
+        if not key:
+            deferreds = []
+            for active in actives:
+                d = self._soledad.delete_doc(active)
+                deferreds.append(d)
+            return gatherResults(deferreds)
+
+        validation = str(ValidationLevels.Weak_Chain)
+        last_audited = 0
+        encr_used = False
+        sign_used = False
+        if len(actives) == 1 and KEY_VERSION_KEY not in key.content:
+            # we can preserve the validation of the key if there is only one
+            # active address for the key
+            validation = key.content[KEY_VALIDATION_KEY]
+            last_audited = key.content[KEY_LAST_AUDITED_AT_KEY]
+            encr_used = key.content[KEY_ENCR_USED_KEY]
+            sign_used = key.content[KEY_SIGN_USED_KEY]
+
+        deferreds = []
+        for active in actives:
+            if KEY_VERSION_KEY in active.content:
+                continue
+
+            active.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+            active.content[KEY_VALIDATION_KEY] = validation
+            active.content[KEY_LAST_AUDITED_AT_KEY] = last_audited
+            active.content[KEY_ENCR_USED_KEY] = encr_used
+            active.content[KEY_SIGN_USED_KEY] = sign_used
+            d = self._soledad.put_doc(active)
+            deferreds.append(d)
+        return gatherResults(deferreds)
+
+    def _migrate_key(self, key):
+        if not key or KEY_VERSION_KEY in key.content:
+            return succeed(None)
+
+        key.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+        del key.content[KEY_VALIDATION_KEY]
+        del key.content[KEY_LAST_AUDITED_AT_KEY]
+        del key.content[KEY_ENCR_USED_KEY]
+        del key.content[KEY_SIGN_USED_KEY]
+        return self._soledad.put_doc(key)
+
+
+def fp2id(fingerprint):
+    KEY_ID_LENGTH = 16
+    return fingerprint[-KEY_ID_LENGTH:]
diff --git a/src/leap/keymanager/tests/__init__.py b/src/leap/keymanager/tests/__init__.py
index cd612c4..d02f187 100644
--- a/src/leap/keymanager/tests/__init__.py
+++ b/src/leap/keymanager/tests/__init__.py
@@ -76,7 +76,7 @@ class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
             return d
 
         # wait for the indexes to be ready for the tear down
-        d = km._wrapper_map[OpenPGPKey].deferred_indexes
+        d = km._wrapper_map[OpenPGPKey].deferred_init
         d.addCallback(get_and_delete_keys)
         d.addCallback(lambda _: self.tearDownEnv())
         d.addCallback(lambda _: self._soledad.close())
-- 
cgit v1.2.3


From 3d544f4a85930c5d1611d193500744fc97f0aee1 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Fri, 18 Dec 2015 20:31:18 +0100
Subject: [feat] Use fingerprints instead of key ids

- Resolves: #7500
---
 src/leap/keymanager/__init__.py              |   6 +-
 src/leap/keymanager/keys.py                  |  21 ++---
 src/leap/keymanager/migrator.py              |   8 +-
 src/leap/keymanager/openpgp.py               |  43 +++++-----
 src/leap/keymanager/tests/__init__.py        |   1 -
 src/leap/keymanager/tests/test_keymanager.py |   5 --
 src/leap/keymanager/tests/test_openpgp.py    | 123 +++++++++++----------------
 src/leap/keymanager/validation.py            |   4 +-
 8 files changed, 91 insertions(+), 120 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 8b3487f..8a4efbe 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -642,7 +642,7 @@ class KeyManager(object):
             else:
                 signature = InvalidSignature(
                     'Failed to verify signature with key %s' %
-                    (pubkey.key_id,))
+                    (pubkey.fingerprint,))
             defer.returnValue((decrypted, signature))
 
         dpriv = self.get_key(address, ktype, private=True)
@@ -741,7 +741,7 @@ class KeyManager(object):
             else:
                 raise InvalidSignature(
                     'Failed to verify signature with key %s' %
-                    (pubkey.key_id,))
+                    (pubkey.fingerprint,))
 
         d = self.get_key(address, ktype, private=False,
                          fetch_remote=fetch_remote)
@@ -804,7 +804,7 @@ class KeyManager(object):
             else:
                 raise KeyNotValidUpgrade(
                     "Key %s can not be upgraded by new key %s"
-                    % (old_key.key_id, key.key_id))
+                    % (old_key.fingerprint, key.fingerprint))
 
         d = _keys.get_key(address, private=key.private)
         d.addErrback(old_key_not_found)
diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index a60c19d..68e3fad 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -48,7 +48,6 @@ logger = logging.getLogger(__name__)
 KEY_VERSION_KEY = 'version'
 KEY_ADDRESS_KEY = 'address'
 KEY_TYPE_KEY = 'type'
-KEY_ID_KEY = 'key_id'
 KEY_FINGERPRINT_KEY = 'fingerprint'
 KEY_DATA_KEY = 'key_data'
 KEY_PRIVATE_KEY = 'private'
@@ -80,16 +79,16 @@ KEYMANAGER_DOC_VERSION = 1
 #
 
 TAGS_PRIVATE_INDEX = 'by-tags-private'
-TYPE_ID_PRIVATE_INDEX = 'by-type-id-private'
+TYPE_FINGERPRINT_PRIVATE_INDEX = 'by-type-fingerprint-private'
 TYPE_ADDRESS_PRIVATE_INDEX = 'by-type-address-private'
 INDEXES = {
     TAGS_PRIVATE_INDEX: [
         KEY_TAGS_KEY,
         'bool(%s)' % KEY_PRIVATE_KEY,
     ],
-    TYPE_ID_PRIVATE_INDEX: [
+    TYPE_FINGERPRINT_PRIVATE_INDEX: [
         KEY_TYPE_KEY,
-        KEY_ID_KEY,
+        KEY_FINGERPRINT_KEY,
         'bool(%s)' % KEY_PRIVATE_KEY,
     ],
     TYPE_ADDRESS_PRIVATE_INDEX: [
@@ -137,7 +136,8 @@ def build_key_from_dict(kClass, key, active=None):
             validation = ValidationLevels.get(active[KEY_VALIDATION_KEY])
         except ValueError:
             logger.error("Not valid validation level (%s) for key %s",
-                         (active[KEY_VALIDATION_KEY], active[KEY_ID_KEY]))
+                         (active[KEY_VALIDATION_KEY],
+                          active[KEY_FINGERPRINT_KEY]))
         last_audited_at = _to_datetime(active[KEY_LAST_AUDITED_AT_KEY])
         encr_used = active[KEY_ENCR_USED_KEY]
         sign_used = active[KEY_SIGN_USED_KEY]
@@ -147,7 +147,6 @@ def build_key_from_dict(kClass, key, active=None):
 
     return kClass(
         key[KEY_ADDRESS_KEY],
-        key_id=key[KEY_ID_KEY],
         fingerprint=key[KEY_FINGERPRINT_KEY],
         key_data=key[KEY_DATA_KEY],
         private=key[KEY_PRIVATE_KEY],
@@ -189,13 +188,12 @@ class EncryptionKey(object):
 
     __metaclass__ = ABCMeta
 
-    def __init__(self, address, key_id="", fingerprint="",
+    def __init__(self, address, fingerprint="",
                  key_data="", private=False, length=0, expiry_date=None,
                  validation=ValidationLevels.Weak_Chain, last_audited_at=None,
                  refreshed_at=None, encr_used=False, sign_used=False):
         # TODO: it should know its own active address
         self.address = address
-        self.key_id = key_id
         self.fingerprint = fingerprint
         self.key_data = key_data
         self.private = private
@@ -221,7 +219,6 @@ class EncryptionKey(object):
         return json.dumps({
             KEY_ADDRESS_KEY: self.address,
             KEY_TYPE_KEY: self.__class__.__name__,
-            KEY_ID_KEY: self.key_id,
             KEY_FINGERPRINT_KEY: self.fingerprint,
             KEY_DATA_KEY: self.key_data,
             KEY_PRIVATE_KEY: self.private,
@@ -244,7 +241,7 @@ class EncryptionKey(object):
         return json.dumps({
             KEY_ADDRESS_KEY: address,
             KEY_TYPE_KEY: self.__class__.__name__ + KEYMANAGER_ACTIVE_TYPE,
-            KEY_ID_KEY: self.key_id,
+            KEY_FINGERPRINT_KEY: self.fingerprint,
             KEY_PRIVATE_KEY: self.private,
             KEY_VALIDATION_KEY: str(self.validation),
             KEY_LAST_AUDITED_AT_KEY: last_audited_at,
@@ -260,7 +257,7 @@ class EncryptionKey(object):
         """
         return u"<%s 0x%s (%s - %s)>" % (
             self.__class__.__name__,
-            self.key_id,
+            self.fingerprint,
             self.address,
             "priv" if self.private else "publ")
 
@@ -519,7 +516,7 @@ class EncryptionScheme(object):
         """
         def log_active_doc(doc):
             logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
-                                       doc.content[KEY_ID_KEY]))
+                                       doc.content[KEY_FINGERPRINT_KEY]))
 
         def cmp_active(d1, d2):
             res = cmp(d1.content[KEY_LAST_AUDITED_AT_KEY],
diff --git a/src/leap/keymanager/migrator.py b/src/leap/keymanager/migrator.py
index b59647a..11cf243 100644
--- a/src/leap/keymanager/migrator.py
+++ b/src/leap/keymanager/migrator.py
@@ -33,7 +33,7 @@ from leap.keymanager.keys import (
 
     KEYMANAGER_DOC_VERSION,
     KEY_VERSION_KEY,
-    KEY_ID_KEY,
+    KEY_FINGERPRINT_KEY,
     KEY_VALIDATION_KEY,
     KEY_LAST_AUDITED_AT_KEY,
     KEY_ENCR_USED_KEY,
@@ -42,6 +42,8 @@ from leap.keymanager.keys import (
 from leap.keymanager.validation import ValidationLevels
 
 
+KEY_ID_KEY = 'key_id'
+
 KeyDocs = namedtuple("KeyDocs", ['key', 'active'])
 
 
@@ -132,6 +134,7 @@ class KeyDocumentsMigrator(object):
         last_audited = 0
         encr_used = False
         sign_used = False
+        fingerprint = key.content[KEY_FINGERPRINT_KEY]
         if len(actives) == 1 and KEY_VERSION_KEY not in key.content:
             # we can preserve the validation of the key if there is only one
             # active address for the key
@@ -146,10 +149,12 @@ class KeyDocumentsMigrator(object):
                 continue
 
             active.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+            active.content[KEY_FINGERPRINT_KEY] = fingerprint
             active.content[KEY_VALIDATION_KEY] = validation
             active.content[KEY_LAST_AUDITED_AT_KEY] = last_audited
             active.content[KEY_ENCR_USED_KEY] = encr_used
             active.content[KEY_SIGN_USED_KEY] = sign_used
+            del active.content[KEY_ID_KEY]
             d = self._soledad.put_doc(active)
             deferreds.append(d)
         return gatherResults(deferreds)
@@ -159,6 +164,7 @@ class KeyDocumentsMigrator(object):
             return succeed(None)
 
         key.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+        del key.content[KEY_ID_KEY]
         del key.content[KEY_VALIDATION_KEY]
         del key.content[KEY_LAST_AUDITED_AT_KEY]
         del key.content[KEY_ENCR_USED_KEY]
diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index 3c8ac1e..0f16296 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -39,10 +39,10 @@ from leap.keymanager.keys import (
     EncryptionScheme,
     is_address,
     build_key_from_dict,
-    TYPE_ID_PRIVATE_INDEX,
+    TYPE_FINGERPRINT_PRIVATE_INDEX,
     TYPE_ADDRESS_PRIVATE_INDEX,
     KEY_ADDRESS_KEY,
-    KEY_ID_KEY,
+    KEY_FINGERPRINT_KEY,
     KEYMANAGER_ACTIVE_TYPE,
 )
 
@@ -122,9 +122,9 @@ class TempGPGWrapper(object):
         # itself is enough to also have the public key in the keyring,
         # and we want to count the keys afterwards.
 
-        privids = map(lambda privkey: privkey.key_id, privkeys)
+        privfps = map(lambda privkey: privkey.fingerprint, privkeys)
         publkeys = filter(
-            lambda pubkey: pubkey.key_id not in privids, publkeys)
+            lambda pubkey: pubkey.fingerprint not in privfps, publkeys)
 
         listkeys = lambda: self._gpg.list_keys()
         listsecretkeys = lambda: self._gpg.list_keys(secret=True)
@@ -213,7 +213,7 @@ class OpenPGPKey(EncryptionKey):
         :rtype: list(str)
         """
         with TempGPGWrapper(keys=[self], gpgbinary=self._gpgbinary) as gpg:
-            res = gpg.list_sigs(self.key_id)
+            res = gpg.list_sigs(self.fingerprint)
             for uid, sigs in res.sigs.iteritems():
                 if _parse_address(uid) in self.address:
                     return sigs
@@ -370,7 +370,7 @@ class OpenPGPScheme(EncryptionScheme):
             leap_assert(
                 address in keydoc.content[KEY_ADDRESS_KEY],
                 'Wrong address in key %s. Expected %s, found %s.'
-                % (keydoc.content[KEY_ID_KEY], address,
+                % (keydoc.content[KEY_FINGERPRINT_KEY], address,
                    keydoc.content[KEY_ADDRESS_KEY]))
             key = build_key_from_dict(OpenPGPKey, keydoc.content,
                                       activedoc.content)
@@ -493,7 +493,7 @@ class OpenPGPScheme(EncryptionScheme):
                 deferreds.append(d)
             return defer.gatherResults(deferreds)
 
-        dk = self._get_key_doc_from_keyid(key.key_id, key.private)
+        dk = self._get_key_doc_from_fingerprint(key.fingerprint, key.private)
         da = self._get_active_doc_from_address(address, key.private)
         d = defer.gatherResults([dk, da])
         d.addCallback(merge_and_put)
@@ -517,8 +517,8 @@ class OpenPGPScheme(EncryptionScheme):
         def get_key_from_active_doc(activedoc):
             if not activedoc:
                 return (None, None)
-            key_id = activedoc.content[KEY_ID_KEY]
-            d = self._get_key_doc_from_keyid(key_id, private)
+            fingerprint = activedoc.content[KEY_FINGERPRINT_KEY]
+            d = self._get_key_doc_from_fingerprint(fingerprint, private)
             d.addCallback(delete_active_if_no_key, activedoc)
             return d
 
@@ -573,17 +573,17 @@ class OpenPGPScheme(EncryptionScheme):
 
         def get_key_docs(_):
             return self._soledad.get_from_index(
-                TYPE_ID_PRIVATE_INDEX,
+                TYPE_FINGERPRINT_PRIVATE_INDEX,
                 self.KEY_TYPE,
-                key.key_id,
+                key.fingerprint,
                 '1' if key.private else '0')
 
         def delete_key(docs):
             if len(docs) == 0:
                 raise errors.KeyNotFound(key)
             elif len(docs) > 1:
-                logger.warning("There is more than one key for key_id %s"
-                               % key.key_id)
+                logger.warning("There is more than one key for fingerprint %s"
+                               % key.fingerprint)
 
             has_deleted = False
             deferreds = []
@@ -597,9 +597,9 @@ class OpenPGPScheme(EncryptionScheme):
             return defer.gatherResults(deferreds)
 
         d = self._soledad.get_from_index(
-            TYPE_ID_PRIVATE_INDEX,
+            TYPE_FINGERPRINT_PRIVATE_INDEX,
             self.ACTIVE_TYPE,
-            key.key_id,
+            key.fingerprint,
             '1' if key.private else '0')
         d.addCallback(delete_docs)
         d.addCallback(get_key_docs)
@@ -659,7 +659,7 @@ class OpenPGPScheme(EncryptionScheme):
             result = yield from_thread(
                 gpg.encrypt,
                 data, pubkey.fingerprint,
-                default_key=sign.key_id if sign else None,
+                default_key=sign.fingerprint if sign else None,
                 passphrase=passphrase, symmetric=False,
                 cipher_algo=cipher_algo)
             # Here we cannot assert for correctness of sig because the sig is
@@ -761,7 +761,7 @@ class OpenPGPScheme(EncryptionScheme):
         # result.fingerprint - contains the fingerprint of the key used to
         #                      sign.
         with TempGPGWrapper(privkey, self._gpgbinary) as gpg:
-            result = gpg.sign(data, default_key=privkey.key_id,
+            result = gpg.sign(data, default_key=privkey.fingerprint,
                               digest_algo=digest_algo, clearsign=clearsign,
                               detach=detach, binary=binary)
             rfprint = privkey.fingerprint
@@ -770,7 +770,7 @@ class OpenPGPScheme(EncryptionScheme):
             if result.fingerprint is None:
                 raise errors.SignFailed(
                     'Failed to sign with key %s: %s' %
-                    (privkey['keyid'], result.stderr))
+                    (privkey['fingerprint'], result.stderr))
             leap_assert(
                 result.fingerprint == kfprint,
                 'Signature and private key fingerprints mismatch: '
@@ -823,11 +823,11 @@ class OpenPGPScheme(EncryptionScheme):
         d.addCallback(self._repair_and_get_doc, self._repair_active_docs)
         return d
 
-    def _get_key_doc_from_keyid(self, key_id, private):
+    def _get_key_doc_from_fingerprint(self, fingerprint, private):
         d = self._soledad.get_from_index(
-            TYPE_ID_PRIVATE_INDEX,
+            TYPE_FINGERPRINT_PRIVATE_INDEX,
             self.KEY_TYPE,
-            key_id,
+            fingerprint,
             '1' if private else '0')
         d.addCallback(self._repair_and_get_doc, self._repair_key_docs)
         return d
@@ -863,7 +863,6 @@ def build_gpg_key(key_info, key_data, gpgbinary=None):
     return OpenPGPKey(
         address,
         gpgbinary=gpgbinary,
-        key_id=key_info['keyid'],
         fingerprint=key_info['fingerprint'],
         key_data=key_data,
         private=True if key_info['type'] == 'sec' else False,
diff --git a/src/leap/keymanager/tests/__init__.py b/src/leap/keymanager/tests/__init__.py
index d02f187..4fbf63e 100644
--- a/src/leap/keymanager/tests/__init__.py
+++ b/src/leap/keymanager/tests/__init__.py
@@ -97,7 +97,6 @@ class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
 
 
 # key 24D18DDF: public key "Leap Test Key <leap@leap.se>"
-KEY_ID = "2F455E2824D18DDF"
 KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF"
 PUBLIC_KEY = """
 -----BEGIN PGP PUBLIC KEY BLOCK-----
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index e4e0d8b..2fe9e4c 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -77,7 +77,6 @@ class KeyManagerUtilTestCase(unittest.TestCase):
     def test_build_key_from_dict(self):
         kdict = {
             'address': [ADDRESS],
-            'key_id': KEY_FINGERPRINT[-16:],
             'fingerprint': KEY_FINGERPRINT,
             'key_data': PUBLIC_KEY,
             'private': False,
@@ -87,7 +86,6 @@ class KeyManagerUtilTestCase(unittest.TestCase):
         }
         adict = {
             'address': ADDRESS,
-            'key_id': KEY_FINGERPRINT[-16:],
             'private': False,
             'last_audited_at': 0,
             'validation': str(ValidationLevels.Weak_Chain),
@@ -98,9 +96,6 @@ class KeyManagerUtilTestCase(unittest.TestCase):
         self.assertEqual(
             kdict['address'], key.address,
             'Wrong data in key.')
-        self.assertEqual(
-            kdict['key_id'], key.key_id,
-            'Wrong data in key.')
         self.assertEqual(
             kdict['fingerprint'], key.fingerprint,
             'Wrong data in key.')
diff --git a/src/leap/keymanager/tests/test_openpgp.py b/src/leap/keymanager/tests/test_openpgp.py
index 6641591..8ed049f 100644
--- a/src/leap/keymanager/tests/test_openpgp.py
+++ b/src/leap/keymanager/tests/test_openpgp.py
@@ -30,7 +30,7 @@ from leap.keymanager import (
     openpgp,
 )
 from leap.keymanager.keys import (
-    TYPE_ID_PRIVATE_INDEX,
+    TYPE_FINGERPRINT_PRIVATE_INDEX,
     TYPE_ADDRESS_PRIVATE_INDEX,
 )
 from leap.keymanager.openpgp import OpenPGPKey
@@ -40,7 +40,6 @@ from leap.keymanager.tests import (
     ADDRESS_2,
     KEY_FINGERPRINT,
     PUBLIC_KEY,
-    KEY_ID,
     PUBLIC_KEY_2,
     PRIVATE_KEY,
     PRIVATE_KEY_2,
@@ -256,39 +255,18 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
 
     @inlineCallbacks
     def test_self_repair_three_keys(self):
+        refreshed_keep = datetime(2007, 1, 1)
+        self._insert_key_docs([datetime(2005, 1, 1),
+                               refreshed_keep,
+                               datetime(2001, 1, 1)])
+        delete_doc = self._mock_delete_doc()
+
         pgp = openpgp.OpenPGPScheme(
             self._soledad, gpgbinary=self.gpg_binary_path)
-        yield pgp.put_ascii_key(PUBLIC_KEY, ADDRESS)
-
-        get_from_index = self._soledad.get_from_index
-        delete_doc = self._soledad.delete_doc
-
-        def my_get_from_index(*args):
-            if (args[0] == TYPE_ID_PRIVATE_INDEX and
-                    args[2] == KEY_ID):
-                k1 = OpenPGPKey(ADDRESS, key_id="1",
-                                refreshed_at=datetime(2005, 1, 1))
-                k2 = OpenPGPKey(ADDRESS, key_id="2",
-                                refreshed_at=datetime(2007, 1, 1))
-                k3 = OpenPGPKey(ADDRESS, key_id="3",
-                                refreshed_at=datetime(2001, 1, 1))
-                d1 = self._soledad.create_doc_from_json(k1.get_json())
-                d2 = self._soledad.create_doc_from_json(k2.get_json())
-                d3 = self._soledad.create_doc_from_json(k3.get_json())
-                return gatherResults([d1, d2, d3])
-            return get_from_index(*args)
-
-        self._soledad.get_from_index = my_get_from_index
-        self._soledad.delete_doc = Mock(return_value=succeed(None))
-
         key = yield pgp.get_key(ADDRESS, private=False)
-
-        try:
-            self.assertEqual(key.key_id, "2")
-            self.assertEqual(self._soledad.delete_doc.call_count, 2)
-        finally:
-            self._soledad.get_from_index = get_from_index
-            self._soledad.delete_doc = delete_doc
+        self.assertEqual(key.refreshed_at, refreshed_keep)
+        self.assertEqual(self.count, 2)
+        self._soledad.delete_doc = delete_doc
 
     @inlineCallbacks
     def test_self_repair_no_keys(self):
@@ -300,8 +278,8 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         delete_doc = self._soledad.delete_doc
 
         def my_get_from_index(*args):
-            if (args[0] == TYPE_ID_PRIVATE_INDEX and
-                    args[2] == KEY_ID):
+            if (args[0] == TYPE_FINGERPRINT_PRIVATE_INDEX and
+                    args[2] == KEY_FINGERPRINT):
                 return succeed([])
             return get_from_index(*args)
 
@@ -319,39 +297,16 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
 
     @inlineCallbacks
     def test_self_repair_put_keys(self):
+        self._insert_key_docs([datetime(2005, 1, 1),
+                               datetime(2007, 1, 1),
+                               datetime(2001, 1, 1)])
+        delete_doc = self._mock_delete_doc()
+
         pgp = openpgp.OpenPGPScheme(
             self._soledad, gpgbinary=self.gpg_binary_path)
-
-        get_from_index = self._soledad.get_from_index
-        delete_doc = self._soledad.delete_doc
-
-        def my_get_from_index(*args):
-            if (args[0] == TYPE_ID_PRIVATE_INDEX and
-                    args[2] == KEY_ID):
-                k1 = OpenPGPKey(ADDRESS, key_id="1",
-                                fingerprint=KEY_FINGERPRINT,
-                                refreshed_at=datetime(2005, 1, 1))
-                k2 = OpenPGPKey(ADDRESS, key_id="2",
-                                fingerprint=KEY_FINGERPRINT,
-                                refreshed_at=datetime(2007, 1, 1))
-                k3 = OpenPGPKey(ADDRESS, key_id="3",
-                                fingerprint=KEY_FINGERPRINT,
-                                refreshed_at=datetime(2001, 1, 1))
-                d1 = self._soledad.create_doc_from_json(k1.get_json())
-                d2 = self._soledad.create_doc_from_json(k2.get_json())
-                d3 = self._soledad.create_doc_from_json(k3.get_json())
-                return gatherResults([d1, d2, d3])
-            return get_from_index(*args)
-
-        self._soledad.get_from_index = my_get_from_index
-        self._soledad.delete_doc = Mock(return_value=succeed(None))
-
-        try:
-            yield pgp.put_ascii_key(PUBLIC_KEY, ADDRESS)
-            self.assertEqual(self._soledad.delete_doc.call_count, 2)
-        finally:
-            self._soledad.get_from_index = get_from_index
-            self._soledad.delete_doc = delete_doc
+        yield pgp.put_ascii_key(PUBLIC_KEY, ADDRESS)
+        self.assertEqual(self.count, 2)
+        self._soledad.delete_doc = delete_doc
 
     @inlineCallbacks
     def test_self_repair_five_active_docs(self):
@@ -364,29 +319,29 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
         def my_get_from_index(*args):
             if (args[0] == TYPE_ADDRESS_PRIVATE_INDEX and
                     args[2] == ADDRESS):
-                k1 = OpenPGPKey(ADDRESS, key_id="1",
+                k1 = OpenPGPKey(ADDRESS, fingerprint="1",
                                 last_audited_at=datetime(2005, 1, 1))
-                k2 = OpenPGPKey(ADDRESS, key_id="2",
+                k2 = OpenPGPKey(ADDRESS, fingerprint="2",
                                 last_audited_at=datetime(2007, 1, 1))
-                k3 = OpenPGPKey(ADDRESS, key_id="3",
+                k3 = OpenPGPKey(ADDRESS, fingerprint="3",
                                 last_audited_at=datetime(2007, 1, 1),
                                 encr_used=True, sign_used=True)
-                k4 = OpenPGPKey(ADDRESS, key_id="4",
+                k4 = OpenPGPKey(ADDRESS, fingerprint="4",
                                 last_audited_at=datetime(2007, 1, 1),
                                 sign_used=True)
-                k5 = OpenPGPKey(ADDRESS, key_id="5",
+                k5 = OpenPGPKey(ADDRESS, fingerprint="5",
                                 last_audited_at=datetime(2007, 1, 1),
                                 encr_used=True)
                 deferreds = []
-                for k in [k1, k2, k3, k4, k5]:
+                for k in (k1, k2, k3, k4, k5):
                     d = self._soledad.create_doc_from_json(
                         k.get_active_json(ADDRESS))
                     deferreds.append(d)
                 return gatherResults(deferreds)
-            elif args[0] == TYPE_ID_PRIVATE_INDEX:
-                key_id = args[2]
-                self.assertEqual(key_id, "3")
-                k = OpenPGPKey(ADDRESS, key_id="3")
+            elif args[0] == TYPE_FINGERPRINT_PRIVATE_INDEX:
+                fingerprint = args[2]
+                self.assertEqual(fingerprint, "3")
+                k = OpenPGPKey(ADDRESS, fingerprint="3")
                 return succeed(
                     [self._soledad.create_doc_from_json(k.get_json())])
             return get_from_index(*args)
@@ -404,3 +359,21 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
     def _assert_key_not_found(self, pgp, address, private=False):
         d = pgp.get_key(address, private=private)
         return self.assertFailure(d, KeyNotFound)
+
+    @inlineCallbacks
+    def _insert_key_docs(self, refreshed_at):
+        for date in refreshed_at:
+            key = OpenPGPKey(ADDRESS, fingerprint=KEY_FINGERPRINT,
+                             refreshed_at=date)
+            yield self._soledad.create_doc_from_json(key.get_json())
+        yield self._soledad.create_doc_from_json(key.get_active_json())
+
+    def _mock_delete_doc(self):
+        delete_doc = self._soledad.delete_doc
+        self.count = 0
+
+        def my_delete_doc(*args):
+            self.count += 1
+            return delete_doc(*args)
+        self._soledad.delete_doc = my_delete_doc
+        return delete_doc
diff --git a/src/leap/keymanager/validation.py b/src/leap/keymanager/validation.py
index 734cfce..8cf96da 100644
--- a/src/leap/keymanager/validation.py
+++ b/src/leap/keymanager/validation.py
@@ -118,7 +118,9 @@ def can_upgrade(new_key, old_key):
         return True
 
     # New key signed by the old key
-    if old_key.key_id in new_key.signatures:
+    # XXX: signatures are using key-ids instead of fingerprints
+    key_id = old_key.fingerprint[-16:]
+    if key_id in new_key.signatures:
         return True
 
     return False
-- 
cgit v1.2.3


From 6222996d0805dfec1fab949b536adb0af08df0be Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Mon, 21 Dec 2015 20:24:52 +0100
Subject: [test] add updater tests

---
 src/leap/keymanager/tests/test_migrator.py | 175 +++++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)
 create mode 100644 src/leap/keymanager/tests/test_migrator.py

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/tests/test_migrator.py b/src/leap/keymanager/tests/test_migrator.py
new file mode 100644
index 0000000..2d9782b
--- /dev/null
+++ b/src/leap/keymanager/tests/test_migrator.py
@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+# test_migrator.py
+# Copyright (C) 2015 LEAP
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Tests for the migrator.
+"""
+
+
+from collections import namedtuple
+from mock import Mock
+from twisted.internet.defer import succeed, inlineCallbacks
+
+from leap.keymanager.migrator import KeyDocumentsMigrator, KEY_ID_KEY
+from leap.keymanager.keys import (
+    TAGS_PRIVATE_INDEX,
+    KEYMANAGER_ACTIVE_TAG,
+    KEYMANAGER_KEY_TAG,
+    KEYMANAGER_DOC_VERSION,
+
+    KEY_ADDRESS_KEY,
+    KEY_UIDS_KEY,
+    KEY_VERSION_KEY,
+    KEY_FINGERPRINT_KEY,
+    KEY_VALIDATION_KEY,
+    KEY_LAST_AUDITED_AT_KEY,
+    KEY_ENCR_USED_KEY,
+    KEY_SIGN_USED_KEY,
+)
+from leap.keymanager.validation import ValidationLevels
+from leap.keymanager.tests import (
+    KeyManagerWithSoledadTestCase,
+    ADDRESS,
+    ADDRESS_2,
+    KEY_FINGERPRINT,
+)
+
+
+class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
+    @inlineCallbacks
+    def test_simple_migration(self):
+        get_from_index = self._soledad.get_from_index
+        delete_doc = self._soledad.delete_doc
+        put_doc = self._soledad.put_doc
+
+        def my_get_from_index(*args):
+            docs = []
+            if (args[0] == TAGS_PRIVATE_INDEX and
+                    args[2] == '0'):
+                SoledadDocument = namedtuple("SoledadDocument", ["content"])
+                if args[1] == KEYMANAGER_KEY_TAG:
+                    docs = [SoledadDocument({
+                        KEY_ADDRESS_KEY: [ADDRESS],
+                        KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                        KEY_FINGERPRINT_KEY: KEY_FINGERPRINT,
+                        KEY_VALIDATION_KEY: str(ValidationLevels.Weak_Chain),
+                        KEY_LAST_AUDITED_AT_KEY: 0,
+                        KEY_ENCR_USED_KEY: True,
+                        KEY_SIGN_USED_KEY: False,
+                    })]
+                if args[1] == KEYMANAGER_ACTIVE_TAG:
+                    docs = [SoledadDocument({
+                        KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                    })]
+            return succeed(docs)
+
+        self._soledad.get_from_index = my_get_from_index
+        self._soledad.delete_doc = Mock(return_value=succeed(None))
+        self._soledad.put_doc = Mock(return_value=succeed(None))
+
+        try:
+            migrator = KeyDocumentsMigrator(self._soledad)
+            yield migrator.migrate()
+            call_list = self._soledad.put_doc.call_args_list
+        finally:
+            self._soledad.get_from_index = get_from_index
+            self._soledad.delete_doc = delete_doc
+            self._soledad.put_doc = put_doc
+
+        self.assertEqual(len(call_list), 2)
+        active = call_list[0][0][0]
+        key = call_list[1][0][0]
+
+        self.assertTrue(KEY_ID_KEY not in active.content)
+        self.assertEqual(active.content[KEY_VERSION_KEY],
+                         KEYMANAGER_DOC_VERSION)
+        self.assertEqual(active.content[KEY_FINGERPRINT_KEY], KEY_FINGERPRINT)
+        self.assertEqual(active.content[KEY_VALIDATION_KEY],
+                         str(ValidationLevels.Weak_Chain))
+        self.assertEqual(active.content[KEY_LAST_AUDITED_AT_KEY], 0)
+        self.assertEqual(active.content[KEY_ENCR_USED_KEY], True)
+        self.assertEqual(active.content[KEY_SIGN_USED_KEY], False)
+
+        self.assertTrue(KEY_ID_KEY not in key.content)
+        self.assertTrue(KEY_ADDRESS_KEY not in key.content)
+        self.assertTrue(KEY_VALIDATION_KEY not in key.content)
+        self.assertTrue(KEY_LAST_AUDITED_AT_KEY not in key.content)
+        self.assertTrue(KEY_ENCR_USED_KEY not in key.content)
+        self.assertTrue(KEY_SIGN_USED_KEY not in key.content)
+        self.assertEqual(key.content[KEY_UIDS_KEY], [ADDRESS])
+
+    @inlineCallbacks
+    def test_two_active_docs(self):
+        get_from_index = self._soledad.get_from_index
+        delete_doc = self._soledad.delete_doc
+        put_doc = self._soledad.put_doc
+
+        def my_get_from_index(*args):
+            docs = []
+            if (args[0] == TAGS_PRIVATE_INDEX and
+                    args[2] == '0'):
+                SoledadDocument = namedtuple("SoledadDocument", ["content"])
+                if args[1] == KEYMANAGER_KEY_TAG:
+                    validation = str(ValidationLevels.Provider_Trust)
+                    docs = [SoledadDocument({
+                        KEY_ADDRESS_KEY: [ADDRESS, ADDRESS_2],
+                        KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                        KEY_FINGERPRINT_KEY: KEY_FINGERPRINT,
+                        KEY_VALIDATION_KEY: validation,
+                        KEY_LAST_AUDITED_AT_KEY: 1984,
+                        KEY_ENCR_USED_KEY: True,
+                        KEY_SIGN_USED_KEY: False,
+                    })]
+                if args[1] == KEYMANAGER_ACTIVE_TAG:
+                    docs = [
+                        SoledadDocument({
+                            KEY_ADDRESS_KEY: ADDRESS,
+                            KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                        }),
+                        SoledadDocument({
+                            KEY_ADDRESS_KEY: ADDRESS_2,
+                            KEY_ID_KEY: KEY_FINGERPRINT[-16:],
+                        }),
+                    ]
+            return succeed(docs)
+
+        self._soledad.get_from_index = my_get_from_index
+        self._soledad.delete_doc = Mock(return_value=succeed(None))
+        self._soledad.put_doc = Mock(return_value=succeed(None))
+
+        try:
+            migrator = KeyDocumentsMigrator(self._soledad)
+            yield migrator.migrate()
+            call_list = self._soledad.put_doc.call_args_list
+        finally:
+            self._soledad.get_from_index = get_from_index
+            self._soledad.delete_doc = delete_doc
+            self._soledad.put_doc = put_doc
+
+        self.assertEqual(len(call_list), 3)
+        for active in [call[0][0] for call in call_list][:2]:
+            self.assertTrue(KEY_ID_KEY not in active.content)
+            self.assertEqual(active.content[KEY_VERSION_KEY],
+                             KEYMANAGER_DOC_VERSION)
+            self.assertEqual(active.content[KEY_FINGERPRINT_KEY],
+                             KEY_FINGERPRINT)
+            self.assertEqual(active.content[KEY_VALIDATION_KEY],
+                             str(ValidationLevels.Weak_Chain))
+            self.assertEqual(active.content[KEY_LAST_AUDITED_AT_KEY], 0)
+            self.assertEqual(active.content[KEY_ENCR_USED_KEY], False)
+            self.assertEqual(active.content[KEY_SIGN_USED_KEY], False)
-- 
cgit v1.2.3


From fdb6e285a97d5af21c7b3bdc02cba6fc21382f74 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Mon, 21 Dec 2015 19:26:55 +0100
Subject: [feat] Make EncryptionKey aware of the active address

---
 src/leap/keymanager/__init__.py              | 35 +++++------
 src/leap/keymanager/keys.py                  | 25 ++++----
 src/leap/keymanager/migrator.py              |  4 ++
 src/leap/keymanager/openpgp.py               | 86 ++++++++++++++++------------
 src/leap/keymanager/tests/test_keymanager.py | 21 ++++---
 src/leap/keymanager/tests/test_openpgp.py    |  2 +-
 6 files changed, 95 insertions(+), 78 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 8a4efbe..99ee163 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -580,7 +580,7 @@ class KeyManager(object):
                 data, pubkey, passphrase, sign=signkey,
                 cipher_algo=cipher_algo)
             pubkey.encr_used = True
-            yield _keys.put_key(pubkey, address)
+            yield _keys.put_key(pubkey)
             defer.returnValue(encrypted)
 
         dpub = self.get_key(address, ktype, private=False,
@@ -637,7 +637,7 @@ class KeyManager(object):
                 signature = pubkey
                 if not pubkey.sign_used:
                     pubkey.sign_used = True
-                    yield _keys.put_key(pubkey, verify)
+                    yield _keys.put_key(pubkey)
                     defer.returnValue((decrypted, signature))
             else:
                 signature = InvalidSignature(
@@ -734,7 +734,7 @@ class KeyManager(object):
             if signed:
                 if not pubkey.sign_used:
                     pubkey.sign_used = True
-                    d = _keys.put_key(pubkey, address)
+                    d = _keys.put_key(pubkey)
                     d.addCallback(lambda _: pubkey)
                     return d
                 return pubkey
@@ -765,20 +765,16 @@ class KeyManager(object):
         _keys = self._wrapper_map[type(key)]
         return _keys.delete_key(key)
 
-    def put_key(self, key, address):
+    def put_key(self, key):
         """
         Put key bound to address in local storage.
 
         :param key: The key to be stored
         :type key: EncryptionKey
-        :param address: address for which this key will be active
-        :type address: str
 
         :return: A Deferred which fires when the key is in the storage, or
-                 which fails with KeyAddressMismatch if address doesn't match
-                 any uid on the key or fails with KeyNotValidUpdate if a key
-                 with the same uid exists and the new one is not a valid update
-                 for it.
+                 which fails with KeyNotValidUpdate if a key with the same
+                 uid exists and the new one is not a valid update for it.
         :rtype: Deferred
 
         :raise UnsupportedKeyTypeError: if invalid key type
@@ -787,11 +783,6 @@ class KeyManager(object):
         self._assert_supported_key_type(ktype)
         _keys = self._wrapper_map[ktype]
 
-        if address not in key.address:
-            return defer.fail(
-                KeyAddressMismatch("UID %s found, but expected %s"
-                                   % (str(key.address), address)))
-
         def old_key_not_found(failure):
             if failure.check(KeyNotFound):
                 return None
@@ -800,13 +791,13 @@ class KeyManager(object):
 
         def check_upgrade(old_key):
             if key.private or can_upgrade(key, old_key):
-                return _keys.put_key(key, address)
+                return _keys.put_key(key)
             else:
                 raise KeyNotValidUpgrade(
                     "Key %s can not be upgraded by new key %s"
                     % (old_key.fingerprint, key.fingerprint))
 
-        d = _keys.get_key(address, private=key.private)
+        d = _keys.get_key(key.address, private=key.private)
         d.addErrback(old_key_not_found)
         d.addCallback(check_upgrade)
         return d
@@ -838,11 +829,11 @@ class KeyManager(object):
         self._assert_supported_key_type(ktype)
         _keys = self._wrapper_map[ktype]
 
-        pubkey, privkey = _keys.parse_ascii_key(key)
+        pubkey, privkey = _keys.parse_ascii_key(key, address)
         pubkey.validation = validation
-        d = self.put_key(pubkey, address)
+        d = self.put_key(pubkey)
         if privkey is not None:
-            d.addCallback(lambda _: self.put_key(privkey, address))
+            d.addCallback(lambda _: self.put_key(privkey))
         return d
 
     @defer.inlineCallbacks
@@ -878,12 +869,12 @@ class KeyManager(object):
         ascii_content = yield self._get_with_combined_ca_bundle(uri)
 
         # XXX parse binary keys
-        pubkey, _ = _keys.parse_ascii_key(ascii_content)
+        pubkey, _ = _keys.parse_ascii_key(ascii_content, address)
         if pubkey is None:
             raise KeyNotFound(uri)
 
         pubkey.validation = validation
-        yield self.put_key(pubkey, address)
+        yield self.put_key(pubkey)
 
     def _assert_supported_key_type(self, ktype):
         """
diff --git a/src/leap/keymanager/keys.py b/src/leap/keymanager/keys.py
index 68e3fad..38d66b5 100644
--- a/src/leap/keymanager/keys.py
+++ b/src/leap/keymanager/keys.py
@@ -46,6 +46,7 @@ logger = logging.getLogger(__name__)
 #
 
 KEY_VERSION_KEY = 'version'
+KEY_UIDS_KEY = 'uids'
 KEY_ADDRESS_KEY = 'address'
 KEY_TYPE_KEY = 'type'
 KEY_FINGERPRINT_KEY = 'fingerprint'
@@ -126,12 +127,14 @@ def build_key_from_dict(kClass, key, active=None):
     :return: An instance of the key.
     :rtype: C{kClass}
     """
+    address = None
     validation = ValidationLevels.Weak_Chain
     last_audited_at = None
     encr_used = False
     sign_used = False
 
     if active:
+        address = active[KEY_ADDRESS_KEY]
         try:
             validation = ValidationLevels.get(active[KEY_VALIDATION_KEY])
         except ValueError:
@@ -146,7 +149,8 @@ def build_key_from_dict(kClass, key, active=None):
     refreshed_at = _to_datetime(key[KEY_REFRESHED_AT_KEY])
 
     return kClass(
-        key[KEY_ADDRESS_KEY],
+        address=address,
+        uids=key[KEY_UIDS_KEY],
         fingerprint=key[KEY_FINGERPRINT_KEY],
         key_data=key[KEY_DATA_KEY],
         private=key[KEY_PRIVATE_KEY],
@@ -188,12 +192,15 @@ class EncryptionKey(object):
 
     __metaclass__ = ABCMeta
 
-    def __init__(self, address, fingerprint="",
+    def __init__(self, address=None, uids=[], fingerprint="",
                  key_data="", private=False, length=0, expiry_date=None,
                  validation=ValidationLevels.Weak_Chain, last_audited_at=None,
                  refreshed_at=None, encr_used=False, sign_used=False):
-        # TODO: it should know its own active address
         self.address = address
+        if not uids and address:
+            self.uids = [address]
+        else:
+            self.uids = uids
         self.fingerprint = fingerprint
         self.key_data = key_data
         self.private = private
@@ -217,7 +224,7 @@ class EncryptionKey(object):
         refreshed_at = _to_unix_time(self.refreshed_at)
 
         return json.dumps({
-            KEY_ADDRESS_KEY: self.address,
+            KEY_UIDS_KEY: self.uids,
             KEY_TYPE_KEY: self.__class__.__name__,
             KEY_FINGERPRINT_KEY: self.fingerprint,
             KEY_DATA_KEY: self.key_data,
@@ -229,7 +236,7 @@ class EncryptionKey(object):
             KEY_TAGS_KEY: [KEYMANAGER_KEY_TAG],
         })
 
-    def get_active_json(self, address):
+    def get_active_json(self):
         """
         Return a JSON string describing this key.
 
@@ -239,7 +246,7 @@ class EncryptionKey(object):
         last_audited_at = _to_unix_time(self.last_audited_at)
 
         return json.dumps({
-            KEY_ADDRESS_KEY: address,
+            KEY_ADDRESS_KEY: self.address,
             KEY_TYPE_KEY: self.__class__.__name__ + KEYMANAGER_ACTIVE_TYPE,
             KEY_FINGERPRINT_KEY: self.fingerprint,
             KEY_PRIVATE_KEY: self.private,
@@ -374,14 +381,12 @@ class EncryptionScheme(object):
         pass
 
     @abstractmethod
-    def put_key(self, key, address):
+    def put_key(self, key):
         """
         Put a key in local storage.
 
         :param key: The key to be stored.
         :type key: EncryptionKey
-        :param address: address for which this key will be active.
-        :type address: str
 
         :return: A Deferred which fires when the key is in the storage.
         :rtype: Deferred
@@ -496,7 +501,7 @@ class EncryptionScheme(object):
         :rtype: Deferred
         """
         def log_key_doc(doc):
-            logger.error("\t%s: %s" % (doc.content[KEY_ADDRESS_KEY],
+            logger.error("\t%s: %s" % (doc.content[KEY_UIDS_KEY],
                                        doc.content[KEY_FINGERPRINT_KEY]))
 
         def cmp_key(d1, d2):
diff --git a/src/leap/keymanager/migrator.py b/src/leap/keymanager/migrator.py
index 11cf243..9e4ae77 100644
--- a/src/leap/keymanager/migrator.py
+++ b/src/leap/keymanager/migrator.py
@@ -32,6 +32,8 @@ from leap.keymanager.keys import (
     KEYMANAGER_ACTIVE_TAG,
 
     KEYMANAGER_DOC_VERSION,
+    KEY_ADDRESS_KEY,
+    KEY_UIDS_KEY,
     KEY_VERSION_KEY,
     KEY_FINGERPRINT_KEY,
     KEY_VALIDATION_KEY,
@@ -164,6 +166,8 @@ class KeyDocumentsMigrator(object):
             return succeed(None)
 
         key.content[KEY_VERSION_KEY] = KEYMANAGER_DOC_VERSION
+        key.content[KEY_UIDS_KEY] = key.content[KEY_ADDRESS_KEY]
+        del key.content[KEY_ADDRESS_KEY]
         del key.content[KEY_ID_KEY]
         del key.content[KEY_VALIDATION_KEY]
         del key.content[KEY_LAST_AUDITED_AT_KEY]
diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index 0f16296..0d5a866 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -41,7 +41,7 @@ from leap.keymanager.keys import (
     build_key_from_dict,
     TYPE_FINGERPRINT_PRIVATE_INDEX,
     TYPE_ADDRESS_PRIVATE_INDEX,
-    KEY_ADDRESS_KEY,
+    KEY_UIDS_KEY,
     KEY_FINGERPRINT_KEY,
     KEYMANAGER_ACTIVE_TYPE,
 )
@@ -200,7 +200,7 @@ class OpenPGPKey(EncryptionKey):
     Base class for OpenPGP keys.
     """
 
-    def __init__(self, address, gpgbinary=None, **kwargs):
+    def __init__(self, address=None, gpgbinary=None, **kwargs):
         self._gpgbinary = gpgbinary
         super(OpenPGPKey, self).__init__(address, **kwargs)
 
@@ -215,7 +215,7 @@ class OpenPGPKey(EncryptionKey):
         with TempGPGWrapper(keys=[self], gpgbinary=self._gpgbinary) as gpg:
             res = gpg.list_sigs(self.fingerprint)
             for uid, sigs in res.sigs.iteritems():
-                if _parse_address(uid) in self.address:
+                if _parse_address(uid) in self.uids:
                     return sigs
 
         return []
@@ -335,8 +335,9 @@ class OpenPGPScheme(EncryptionScheme):
                     key = gpg.list_keys(secret=secret).pop()
                     openpgp_key = self._build_key_from_gpg(
                         key,
-                        gpg.export_keys(key['fingerprint'], secret=secret))
-                    d = self.put_key(openpgp_key, address)
+                        gpg.export_keys(key['fingerprint'], secret=secret),
+                        address)
+                    d = self.put_key(openpgp_key)
                     deferreds.append(d)
                 yield defer.gatherResults(deferreds)
 
@@ -368,10 +369,10 @@ class OpenPGPScheme(EncryptionScheme):
             if keydoc is None:
                 raise errors.KeyNotFound(address)
             leap_assert(
-                address in keydoc.content[KEY_ADDRESS_KEY],
+                address in keydoc.content[KEY_UIDS_KEY],
                 'Wrong address in key %s. Expected %s, found %s.'
                 % (keydoc.content[KEY_FINGERPRINT_KEY], address,
-                   keydoc.content[KEY_ADDRESS_KEY]))
+                   keydoc.content[KEY_UIDS_KEY]))
             key = build_key_from_dict(OpenPGPKey, keydoc.content,
                                       activedoc.content)
             key._gpgbinary = self._gpgbinary
@@ -381,13 +382,15 @@ class OpenPGPScheme(EncryptionScheme):
         d.addCallback(build_key)
         return d
 
-    def parse_ascii_key(self, key_data):
+    def parse_ascii_key(self, key_data, address=None):
         """
         Parses an ascii armored key (or key pair) data and returns
         the OpenPGPKey keys.
 
         :param key_data: the key data to be parsed.
         :type key_data: str or unicode
+        :param address: Active address for the key.
+        :type address: str
 
         :returns: the public key and private key (if applies) for that data.
         :rtype: (public, private) -> tuple(OpenPGPKey, OpenPGPKey)
@@ -408,12 +411,13 @@ class OpenPGPScheme(EncryptionScheme):
         openpgp_privkey = None
         if privkey:
             # build private key
-            openpgp_privkey = self._build_key_from_gpg(priv_info, privkey)
+            openpgp_privkey = self._build_key_from_gpg(priv_info, privkey,
+                                                       address)
             leap_check(pub_info['fingerprint'] == priv_info['fingerprint'],
                        'Fingerprints for public and private key differ.',
                        errors.KeyFingerprintMismatch)
         # build public key
-        openpgp_pubkey = self._build_key_from_gpg(pub_info, pubkey)
+        openpgp_pubkey = self._build_key_from_gpg(pub_info, pubkey, address)
 
         return (openpgp_pubkey, openpgp_privkey)
 
@@ -433,12 +437,13 @@ class OpenPGPScheme(EncryptionScheme):
 
         openpgp_privkey = None
         try:
-            openpgp_pubkey, openpgp_privkey = self.parse_ascii_key(key_data)
+            openpgp_pubkey, openpgp_privkey = self.parse_ascii_key(
+                key_data, address)
         except (errors.KeyAddressMismatch, errors.KeyFingerprintMismatch) as e:
             return defer.fail(e)
 
         def put_key(_, key):
-            return self.put_key(key, address)
+            return self.put_key(key)
 
         d = defer.succeed(None)
         if openpgp_pubkey is not None:
@@ -447,14 +452,12 @@ class OpenPGPScheme(EncryptionScheme):
             d.addCallback(put_key, openpgp_privkey)
         return d
 
-    def put_key(self, key, address):
+    def put_key(self, key):
         """
         Put C{key} in local storage.
 
         :param key: The key to be stored.
         :type key: OpenPGPKey
-        :param address: address for which this key will be active.
-        :type address: str
 
         :return: A Deferred which fires when the key is in the storage.
         :rtype: Deferred
@@ -471,31 +474,36 @@ class OpenPGPScheme(EncryptionScheme):
 
             key.merge(oldkey)
             keydoc.set_json(key.get_json())
-            deferred_key = self._soledad.put_doc(keydoc)
-
-            active_json = key.get_active_json(address)
-            if activedoc:
-                activedoc.set_json(active_json)
-                deferred_active = self._soledad.put_doc(activedoc)
-            else:
-                deferred_active = self._soledad.create_doc_from_json(
-                    active_json)
-
-            return defer.gatherResults([deferred_key, deferred_active])
+            d = self._soledad.put_doc(keydoc)
+            d.addCallback(put_active, activedoc)
+            return d
 
         def put_new_key(activedoc):
             deferreds = []
             if activedoc:
                 d = self._soledad.delete_doc(activedoc)
                 deferreds.append(d)
-            for json in [key.get_json(), key.get_active_json(address)]:
+            for json in [key.get_json(), key.get_active_json()]:
                 d = self._soledad.create_doc_from_json(json)
                 deferreds.append(d)
             return defer.gatherResults(deferreds)
 
-        dk = self._get_key_doc_from_fingerprint(key.fingerprint, key.private)
-        da = self._get_active_doc_from_address(address, key.private)
-        d = defer.gatherResults([dk, da])
+        def put_active(_, activedoc):
+            active_json = key.get_active_json()
+            if activedoc:
+                activedoc.set_json(active_json)
+                d = self._soledad.put_doc(activedoc)
+            else:
+                d = self._soledad.create_doc_from_json(active_json)
+            return d
+
+        def get_active_doc(keydoc):
+            d = self._get_active_doc_from_address(key.address, key.private)
+            d.addCallback(lambda activedoc: (keydoc, activedoc))
+            return d
+
+        d = self._get_key_doc_from_fingerprint(key.fingerprint, key.private)
+        d.addCallback(get_active_doc)
         d.addCallback(merge_and_put)
         return d
 
@@ -533,7 +541,7 @@ class OpenPGPScheme(EncryptionScheme):
         d.addCallback(get_key_from_active_doc)
         return d
 
-    def _build_key_from_gpg(self, key, key_data):
+    def _build_key_from_gpg(self, key, key_data, address=None):
         """
         Build an OpenPGPKey for C{address} based on C{key} from
         local gpg storage.
@@ -541,6 +549,8 @@ class OpenPGPScheme(EncryptionScheme):
         ASCII armored GPG key data has to be queried independently in this
         wrapper, so we receive it in C{key_data}.
 
+        :param address: Active address for the key.
+        :type address: str
         :param key: Key obtained from GPG storage.
         :type key: dict
         :param key_data: Key data obtained from GPG storage.
@@ -548,7 +558,7 @@ class OpenPGPScheme(EncryptionScheme):
         :return: An instance of the key.
         :rtype: OpenPGPKey
         """
-        return build_gpg_key(key, key_data, self._gpgbinary)
+        return build_gpg_key(key, key_data, address, self._gpgbinary)
 
     def delete_key(self, key):
         """
@@ -852,16 +862,20 @@ def process_ascii_key(key_data, gpgbinary, secret=False):
     return info, key
 
 
-def build_gpg_key(key_info, key_data, gpgbinary=None):
+def build_gpg_key(key_info, key_data, address=None, gpgbinary=None):
     expiry_date = None
     if key_info['expires']:
         expiry_date = datetime.fromtimestamp(int(key_info['expires']))
-    address = []
+    uids = []
     for uid in key_info['uids']:
-        address.append(_parse_address(uid))
+        uids.append(_parse_address(uid))
+    if address and address not in uids:
+        raise errors.KeyAddressMismatch("UIDs %s found, but expected %s"
+                                        % (str(uids), address))
 
     return OpenPGPKey(
-        address,
+        address=address,
+        uids=uids,
         gpgbinary=gpgbinary,
         fingerprint=key_info['fingerprint'],
         key_data=key_data,
diff --git a/src/leap/keymanager/tests/test_keymanager.py b/src/leap/keymanager/tests/test_keymanager.py
index 2fe9e4c..6347d56 100644
--- a/src/leap/keymanager/tests/test_keymanager.py
+++ b/src/leap/keymanager/tests/test_keymanager.py
@@ -76,7 +76,7 @@ class KeyManagerUtilTestCase(unittest.TestCase):
 
     def test_build_key_from_dict(self):
         kdict = {
-            'address': [ADDRESS],
+            'uids': [ADDRESS],
             'fingerprint': KEY_FINGERPRINT,
             'key_data': PUBLIC_KEY,
             'private': False,
@@ -94,7 +94,7 @@ class KeyManagerUtilTestCase(unittest.TestCase):
         }
         key = build_key_from_dict(OpenPGPKey, kdict, adict)
         self.assertEqual(
-            kdict['address'], key.address,
+            kdict['uids'], key.uids,
             'Wrong data in key.')
         self.assertEqual(
             kdict['fingerprint'], key.fingerprint,
@@ -117,6 +117,9 @@ class KeyManagerUtilTestCase(unittest.TestCase):
         self.assertEqual(
             datetime.fromtimestamp(kdict['refreshed_at']), key.refreshed_at,
             'Wrong data in key.')
+        self.assertEqual(
+            adict['address'], key.address,
+            'Wrong data in key.')
         self.assertEqual(
             ValidationLevels.get(adict['validation']), key.validation,
             'Wrong data in key.')
@@ -137,12 +140,12 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         # get public keys
         keys = yield km.get_all_keys(False)
         self.assertEqual(len(keys), 1, 'Wrong number of keys')
-        self.assertTrue(ADDRESS in keys[0].address)
+        self.assertTrue(ADDRESS in keys[0].uids)
         self.assertFalse(keys[0].private)
         # get private keys
         keys = yield km.get_all_keys(True)
         self.assertEqual(len(keys), 1, 'Wrong number of keys')
-        self.assertTrue(ADDRESS in keys[0].address)
+        self.assertTrue(ADDRESS in keys[0].uids)
         self.assertTrue(keys[0].private)
 
     @defer.inlineCallbacks
@@ -153,7 +156,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         key = yield km.get_key(ADDRESS, OpenPGPKey, private=False,
                                fetch_remote=False)
         self.assertTrue(key is not None)
-        self.assertTrue(ADDRESS in key.address)
+        self.assertTrue(ADDRESS in key.uids)
         self.assertEqual(
             key.fingerprint.lower(), KEY_FINGERPRINT.lower())
         self.assertFalse(key.private)
@@ -166,7 +169,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         key = yield km.get_key(ADDRESS, OpenPGPKey, private=True,
                                fetch_remote=False)
         self.assertTrue(key is not None)
-        self.assertTrue(ADDRESS in key.address)
+        self.assertTrue(ADDRESS in key.uids)
         self.assertEqual(
             key.fingerprint.lower(), KEY_FINGERPRINT.lower())
         self.assertTrue(key.private)
@@ -231,7 +234,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         key = yield self._fetch_key(km, ADDRESS, PUBLIC_KEY)
         self.assertIsInstance(key, OpenPGPKey)
-        self.assertTrue(ADDRESS in key.address)
+        self.assertTrue(ADDRESS in key.uids)
         self.assertEqual(key.validation, ValidationLevels.Provider_Trust)
 
     @defer.inlineCallbacks
@@ -243,7 +246,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
 
         key = yield self._fetch_key(km, ADDRESS_OTHER, PUBLIC_KEY_OTHER)
         self.assertIsInstance(key, OpenPGPKey)
-        self.assertTrue(ADDRESS_OTHER in key.address)
+        self.assertTrue(ADDRESS_OTHER in key.uids)
         self.assertEqual(key.validation, ValidationLevels.Weak_Chain)
 
     def _fetch_key(self, km, address, key):
@@ -273,7 +276,7 @@ class KeyManagerKeyManagementTestCase(KeyManagerWithSoledadTestCase):
         yield km.put_raw_key(PUBLIC_KEY, OpenPGPKey, ADDRESS)
         key = yield km.get_key(ADDRESS, OpenPGPKey)
         self.assertIsInstance(key, OpenPGPKey)
-        self.assertTrue(ADDRESS in key.address)
+        self.assertTrue(ADDRESS in key.uids)
 
     @defer.inlineCallbacks
     def test_fetch_uri_ascii_key(self):
diff --git a/src/leap/keymanager/tests/test_openpgp.py b/src/leap/keymanager/tests/test_openpgp.py
index 8ed049f..0e5f6be 100644
--- a/src/leap/keymanager/tests/test_openpgp.py
+++ b/src/leap/keymanager/tests/test_openpgp.py
@@ -335,7 +335,7 @@ class OpenPGPCryptoTestCase(KeyManagerWithSoledadTestCase):
                 deferreds = []
                 for k in (k1, k2, k3, k4, k5):
                     d = self._soledad.create_doc_from_json(
-                        k.get_active_json(ADDRESS))
+                        k.get_active_json())
                     deferreds.append(d)
                 return gatherResults(deferreds)
             elif args[0] == TYPE_FINGERPRINT_PRIVATE_INDEX:
-- 
cgit v1.2.3


From 1b4d08537e2a526e8f5a76dbe7c7f6d979025296 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Thu, 21 Jan 2016 18:48:23 +0100
Subject: [feat] update usage only if needed

During encryption we where updating 'enc_used' in the key without
checking if it was already set.
---
 src/leap/keymanager/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 99ee163..9e3b6ee 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -579,8 +579,9 @@ class KeyManager(object):
             encrypted = yield _keys.encrypt(
                 data, pubkey, passphrase, sign=signkey,
                 cipher_algo=cipher_algo)
-            pubkey.encr_used = True
-            yield _keys.put_key(pubkey)
+            if not pubkey.encr_used:
+                pubkey.encr_used = True
+                yield _keys.put_key(pubkey)
             defer.returnValue(encrypted)
 
         dpub = self.get_key(address, ktype, private=False,
-- 
cgit v1.2.3


From d44d72c816ebc668931071cf5bd8e37e0be32e05 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Thu, 25 Feb 2016 11:34:45 -0600
Subject: [test] refactor key deletion tests

---
 src/leap/keymanager/tests/__init__.py | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/tests/__init__.py b/src/leap/keymanager/tests/__init__.py
index 4fbf63e..20d05e8 100644
--- a/src/leap/keymanager/tests/__init__.py
+++ b/src/leap/keymanager/tests/__init__.py
@@ -54,6 +54,14 @@ class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
     def tearDown(self):
         km = self._key_manager()
 
+        # wait for the indexes to be ready for the tear down
+        d = km._wrapper_map[OpenPGPKey].deferred_init
+        d.addCallback(lambda _: self.delete_all_keys(km))
+        d.addCallback(lambda _: self.tearDownEnv())
+        d.addCallback(lambda _: self._soledad.close())
+        return d
+
+    def delete_all_keys(self, km):
         def delete_keys(keys):
             deferreds = []
             for key in keys:
@@ -61,26 +69,18 @@ class KeyManagerWithSoledadTestCase(unittest.TestCase, BaseLeapTest):
                 deferreds.append(d)
             return gatherResults(deferreds)
 
-        def get_and_delete_keys(_):
-            deferreds = []
-            for private in [True, False]:
-                d = km.get_all_keys(private=private)
-                d.addCallback(delete_keys)
-                d.addCallback(check_deleted, private)
-                deferreds.append(d)
-            return gatherResults(deferreds)
-
         def check_deleted(_, private):
             d = km.get_all_keys(private=private)
             d.addCallback(lambda keys: self.assertEqual(keys, []))
             return d
 
-        # wait for the indexes to be ready for the tear down
-        d = km._wrapper_map[OpenPGPKey].deferred_init
-        d.addCallback(get_and_delete_keys)
-        d.addCallback(lambda _: self.tearDownEnv())
-        d.addCallback(lambda _: self._soledad.close())
-        return d
+        deferreds = []
+        for private in [True, False]:
+            d = km.get_all_keys(private=private)
+            d.addCallback(delete_keys)
+            d.addCallback(check_deleted, private)
+            deferreds.append(d)
+        return gatherResults(deferreds)
 
     def _key_manager(self, user=ADDRESS, url='', token=None,
                      ca_cert_path=None):
-- 
cgit v1.2.3


From 288c775034d4c18846518a11677e4580a91cf437 Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Sun, 20 Mar 2016 18:44:23 +0100
Subject: [bug] Return KeyNotFound Failure if not valid key is given to
 put_raw_key

- Resolves: #7974
---
 src/leap/keymanager/__init__.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 9e3b6ee..1106c23 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -820,9 +820,10 @@ class KeyManager(object):
 
         :return: A Deferred which fires when the key is in the storage, or
                  which fails with KeyAddressMismatch if address doesn't match
-                 any uid on the key or fails with KeyNotValidUpdate if a key
-                 with the same uid exists and the new one is not a valid update
-                 for it.
+                 any uid on the key or fails with KeyNotFound if no OpenPGP
+                 material was found in key or fails with KeyNotValidUpdate if a
+                 key with the same uid exists and the new one is not a valid
+                 update for it.
         :rtype: Deferred
 
         :raise UnsupportedKeyTypeError: if invalid key type
@@ -831,6 +832,10 @@ class KeyManager(object):
         _keys = self._wrapper_map[ktype]
 
         pubkey, privkey = _keys.parse_ascii_key(key, address)
+
+        if pubkey is None:
+            return defer.fail(KeyNotFound(key))
+
         pubkey.validation = validation
         d = self.put_key(pubkey)
         if privkey is not None:
-- 
cgit v1.2.3


From 495efc574a88d33c5528a1211faf8ca1e684773e Mon Sep 17 00:00:00 2001
From: Kali Kaneko <kali@leap.se>
Date: Fri, 1 Apr 2016 17:25:58 -0400
Subject: [pkg] update to versioneer 0.16

---
 src/leap/keymanager/__init__.py |   4 +
 src/leap/keymanager/_version.py | 544 ++++++++++++++++++++++++++++++----------
 2 files changed, 415 insertions(+), 133 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 1106c23..91cd2f8 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -921,3 +921,7 @@ def _get_domain(url):
     :rtype: str
     """
     return urlparse(url).hostname
+
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions
diff --git a/src/leap/keymanager/_version.py b/src/leap/keymanager/_version.py
index 5153a9b..b28c697 100644
--- a/src/leap/keymanager/_version.py
+++ b/src/leap/keymanager/_version.py
@@ -1,73 +1,157 @@
+
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (build by setup.py sdist) and build
+# feature). Distribution tarballs (built by setup.py sdist) and build
 # directories (produced by setup.py build) will contain a much shorter file
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.7+ (https://github.com/warner/python-versioneer)
+# versioneer-0.16 (https://github.com/warner/python-versioneer)
+
+"""Git implementation of _version.py."""
 
-# these strings will be replaced by git during git-archive
+import errno
+import os
+import re
 import subprocess
 import sys
-import re
-import os.path
 
-IN_LONG_VERSION_PY = True
 
-git_refnames = "$Format:%d$"
-git_full = "$Format:%H$"
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "$Format:%d$"
+    git_full = "$Format:%H$"
+    keywords = {"refnames": git_refnames, "full": git_full}
+    return keywords
 
 
-def run_command(args, cwd=None, verbose=False):
-    try:
-        # remember shell=False, so use git.cmd on windows, not just git
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        e = sys.exc_info()[1]
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "pep440"
+    cfg.tag_prefix = ""
+    cfg.parentdir_prefix = "None"
+    cfg.versionfile_source = "src/leap/keymanager/_version.py"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None
+    else:
         if verbose:
-            print("unable to run %s" % args[0])
-            print(e)
+            print("unable to find command, tried %s" % (commands,))
         return None
     stdout = p.communicate()[0].strip()
-    if sys.version >= '3':
+    if sys.version_info[0] >= 3:
         stdout = stdout.decode()
     if p.returncode != 0:
         if verbose:
-            print("unable to run %s (error)" % args[0])
+            print("unable to run %s (error)" % dispcmd)
         return None
     return stdout
 
 
-def get_expanded_variables(versionfile_source):
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes
+    both the project name and a version string.
+    """
+    dirname = os.path.basename(root)
+    if not dirname.startswith(parentdir_prefix):
+        if verbose:
+            print("guessing rootdir is '%s', but '%s' doesn't start with "
+                  "prefix '%s'" % (root, dirname, parentdir_prefix))
+        raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+    return {"version": dirname[len(parentdir_prefix):],
+            "full-revisionid": None,
+            "dirty": False, "error": None}
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
     # the code embedded in _version.py can just fetch the value of these
-    # variables. When used from setup.py, we don't want to import
-    # _version.py, so we do it with a regexp instead. This function is not
-    # used from _version.py.
-    variables = {}
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
     try:
-        f = open(versionfile_source, "r")
+        f = open(versionfile_abs, "r")
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["refnames"] = mo.group(1)
+                    keywords["refnames"] = mo.group(1)
             if line.strip().startswith("git_full ="):
                 mo = re.search(r'=\s*"(.*)"', line)
                 if mo:
-                    variables["full"] = mo.group(1)
+                    keywords["full"] = mo.group(1)
         f.close()
     except EnvironmentError:
         pass
-    return variables
+    return keywords
 
 
-def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
-    refnames = variables["refnames"].strip()
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    refnames = keywords["refnames"].strip()
     if refnames.startswith("$Format"):
         if verbose:
-            print("variables are unexpanded, not using")
-        return {}  # unexpanded, so not in an unpacked git-archive tarball
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
     refs = set([r.strip() for r in refnames.strip("()").split(",")])
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
@@ -83,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
         # "stabilization", as well as "HEAD" and "master".
         tags = set([r for r in refs if re.search(r'\d', r)])
         if verbose:
-            print("discarding '%s', no digits" % ",".join(refs - tags))
+            print("discarding '%s', no digits" % ",".join(refs-tags))
     if verbose:
         print("likely tags: %s" % ",".join(sorted(tags)))
     for ref in sorted(tags):
@@ -93,114 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
             if verbose:
                 print("picking %s" % r)
             return {"version": r,
-                    "full": variables["full"].strip()}
-    # no suitable tags, so we use the full revision id
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None
+                    }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
-        print("no suitable tags, using full revision id")
-    return {"version": variables["full"].strip(),
-            "full": variables["full"].strip()}
-
-
-def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
-    # this runs 'git' from the root of the source tree. That either means
-    # someone ran a setup.py command (and this code is in versioneer.py, so
-    # IN_LONG_VERSION_PY=False, thus the containing directory is the root of
-    # the source tree), or someone ran a project-specific entry point (and
-    # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
-    # containing directory is somewhere deeper in the source tree). This only
-    # gets called if the git-archive 'subst' variables were *not* expanded,
-    # and _version.py hasn't already been rewritten with a short version
-    # string, meaning we're inside a checked out source tree.
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags"}
 
-    try:
-        here = os.path.abspath(__file__)
-    except NameError:
-        # some py2exe/bbfreeze/non-CPython implementations don't do __file__
-        return {}  # not always correct
-
-    # versionfile_source is the relative path from the top of the source tree
-    # (where the .git directory might live) to this file. Invert this to find
-    # the root from __file__.
-    root = here
-    if IN_LONG_VERSION_PY:
-        for i in range(len(versionfile_source.split("/"))):
-            root = os.path.dirname(root)
-    else:
-        root = os.path.dirname(here)
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
     if not os.path.exists(os.path.join(root, ".git")):
         if verbose:
             print("no .git in %s" % root)
-        return {}
+        raise NotThisMethod("no .git directory")
 
-    GIT = "git"
+    GITS = ["git"]
     if sys.platform == "win32":
-        GIT = "git.cmd"
-    stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
-                         cwd=root)
-    if stdout is None:
-        return {}
-    if not stdout.startswith(tag_prefix):
-        if verbose:
-            print("tag '%s' doesn't start with prefix '%s'" %
-                  (stdout, tag_prefix))
-        return {}
-    tag = stdout[len(tag_prefix):]
-    stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
-    if stdout is None:
-        return {}
-    full = stdout.strip()
-    if tag.endswith("-dirty"):
-        full += "-dirty"
-    return {"version": tag, "full": full}
-
-
-def versions_from_parentdir(parentdir_prefix, versionfile_source,
-                            verbose=False):
-    if IN_LONG_VERSION_PY:
-        # We're running from _version.py. If it's from a source tree
-        # (execute-in-place), we can work upwards to find the root of the
-        # tree, and then check the parent directory for a version string. If
-        # it's in an installed application, there's no hope.
-        try:
-            here = os.path.abspath(__file__)
-        except NameError:
-            # py2exe/bbfreeze/non-CPython don't have __file__
-            return {}  # without __file__, we have no hope
+        GITS = ["git.cmd", "git.exe"]
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out = run_command(GITS, ["describe", "--tags", "--dirty",
+                                      "--always", "--long",
+                                      "--match", "%s*" % tag_prefix],
+                               cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"]}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
+
+    try:
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
+
+    try:
+        root = os.path.realpath(__file__)
         # versionfile_source is the relative path from the top of the source
-        # tree to _version.py. Invert this to find the root from __file__.
-        root = here
-        for i in range(len(versionfile_source.split("/"))):
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
-    else:
-        # we're running from versioneer.py, which means we're running from
-        # the setup.py in a source tree. sys.argv[0] is setup.py in the root.
-        here = os.path.abspath(sys.argv[0])
-        root = os.path.dirname(here)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree"}
 
-    # Source tarballs conventionally unpack into a directory that includes
-    # both the project name and a version string.
-    dirname = os.path.basename(root)
-    if not dirname.startswith(parentdir_prefix):
-        if verbose:
-            print("guessing rootdir is '%s', but '%s' doesn't start "
-                  "with prefix '%s'" %
-                  (root, dirname, parentdir_prefix))
-        return None
-    return {"version": dirname[len(parentdir_prefix):], "full": ""}
-
-tag_prefix = ""
-parentdir_prefix = "leap.keymanager-"
-versionfile_source = "src/leap/keymanager/_version.py"
-
-
-def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
-    variables = {"refnames": git_refnames, "full": git_full}
-    ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
-    if not ver:
-        ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
-    if not ver:
-        ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
-                                      verbose)
-    if not ver:
-        ver = default
-    return ver
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version"}
-- 
cgit v1.2.3


From 86d89807b1ce0eeaaf422143ad9805237c6cb407 Mon Sep 17 00:00:00 2001
From: Bruno Wagner <bwagner@riseup.net>
Date: Mon, 11 Apr 2016 11:57:58 -0300
Subject: [style] Removed duplicated import

There was a duplicate import for get_versions, that was not at
the top of the file, that caused a pep warning and was fixed
in this commit
---
 src/leap/keymanager/__init__.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index 91cd2f8..ce49667 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -922,6 +922,5 @@ def _get_domain(url):
     """
     return urlparse(url).hostname
 
-from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions
-- 
cgit v1.2.3


From f719b8546a214b6bd6ecbef715c5f34b42ad8d52 Mon Sep 17 00:00:00 2001
From: "Kali Kaneko (leap communications)" <kali@leap.se>
Date: Mon, 11 Apr 2016 12:14:01 -0400
Subject: [bug] delete versioneer duplicated block

---
 src/leap/keymanager/__init__.py | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/__init__.py b/src/leap/keymanager/__init__.py
index ce49667..1106c23 100644
--- a/src/leap/keymanager/__init__.py
+++ b/src/leap/keymanager/__init__.py
@@ -921,6 +921,3 @@ def _get_domain(url):
     :rtype: str
     """
     return urlparse(url).hostname
-
-__version__ = get_versions()['version']
-del get_versions
-- 
cgit v1.2.3


From fe46507c44fbfb9c8ed72799d5e8bfb0ac32a10c Mon Sep 17 00:00:00 2001
From: Ruben Pollan <meskio@sindominio.net>
Date: Fri, 8 Apr 2016 18:22:14 +0200
Subject: [feat] reduce log level for encrypt/decrypt errors

* Related: #8022
---
 src/leap/keymanager/openpgp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'src/leap/keymanager')

diff --git a/src/leap/keymanager/openpgp.py b/src/leap/keymanager/openpgp.py
index 0d5a866..a843261 100644
--- a/src/leap/keymanager/openpgp.py
+++ b/src/leap/keymanager/openpgp.py
@@ -680,7 +680,7 @@ class OpenPGPScheme(EncryptionScheme):
                 self._assert_gpg_result_ok(result)
                 defer.returnValue(result.data)
             except errors.GPGError as e:
-                logger.error('Failed to decrypt: %s.' % str(e))
+                logger.warning('Failed to encrypt: %s.' % str(e))
                 raise errors.EncryptError()
 
     @defer.inlineCallbacks
@@ -726,7 +726,7 @@ class OpenPGPScheme(EncryptionScheme):
 
                 defer.returnValue((result.data, sign_valid))
             except errors.GPGError as e:
-                logger.error('Failed to decrypt: %s.' % str(e))
+                logger.warning('Failed to decrypt: %s.' % str(e))
                 raise errors.DecryptError(str(e))
 
     def is_encrypted(self, data):
-- 
cgit v1.2.3