From 2ed8f2b64f3093255d03f282917eb88aac9ab124 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 30 Nov 2015 12:21:56 -0400 Subject: [feat] use uuid in events, multi-user aware - Resolves: #7656 - Releases: 0.8.0 --- client/src/leap/soledad/client/api.py | 6 +++--- client/src/leap/soledad/client/http_target/__init__.py | 1 + client/src/leap/soledad/client/http_target/api.py | 5 +++++ client/src/leap/soledad/client/http_target/fetch.py | 12 +++++++++--- client/src/leap/soledad/client/http_target/send.py | 13 ++++++++++--- 5 files changed, 28 insertions(+), 9 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 8c5f7f1b..0d0d636c 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -102,11 +102,11 @@ class Soledad(object): soledad starts to retrieve keys from server. SOLEDAD_DONE_DOWNLOADING_KEYS: emitted during bootstrap sequence when soledad finishes downloading keys from server. - SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when - there's indeed new data to be synchronized between local database - replica and server's replica. SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has finished synchronizing with remote replica. + SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when + there's indeed new data to be synchronized between local database + replica and server's replica. --- not used right now. """ implements(soledad_interfaces.ILocalStorage, soledad_interfaces.ISyncableStorage, diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py index 498fb6e7..a16531ef 100644 --- a/client/src/leap/soledad/client/http_target/__init__.py +++ b/client/src/leap/soledad/client/http_target/__init__.py @@ -79,6 +79,7 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): self._url = str(url) + "/sync-from/" + str(source_replica_uid) self.source_replica_uid = source_replica_uid self._auth_header = None + self._uuid = None self.set_creds(creds) self._crypto = crypto self._sync_db = sync_db diff --git a/client/src/leap/soledad/client/http_target/api.py b/client/src/leap/soledad/client/http_target/api.py index dcc762f6..94354092 100644 --- a/client/src/leap/soledad/client/http_target/api.py +++ b/client/src/leap/soledad/client/http_target/api.py @@ -40,6 +40,10 @@ class SyncTargetAPI(SyncTarget): self._sync_decr_pool.stop() yield self._http.close() + @property + def uuid(self): + return self._uuid + def set_creds(self, creds): """ Update credentials. @@ -49,6 +53,7 @@ class SyncTargetAPI(SyncTarget): """ uuid = creds['token']['uuid'] token = creds['token']['token'] + self._uuid = uuid auth = '%s:%s' % (uuid, token) b64_token = base64.b64encode(auth) self._auth_header = {'Authorization': ['Token %s' % b64_token]} diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index 65e576d9..087da3a3 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -39,6 +39,12 @@ class HTTPDocFetcher(object): So we parse, decrypt and insert locally as they arrive. """ + # The uuid of the local replica. + # Any class inheriting from this one should provide a meaningful attribute + # if the sync status event is meant to be used somewhere else. + + uuid = 'undefined' + @defer.inlineCallbacks def _receive_docs(self, last_known_generation, last_known_trans_id, ensure_callback, sync_id, defer_decryption): @@ -176,7 +182,7 @@ class HTTPDocFetcher(object): # end of symmetric decryption # ------------------------------------------------------------- self._received_docs += 1 - _emit_receive_status(self._received_docs, total) + _emit_receive_status(self.uuid, self._received_docs, total) return number_of_changes, new_generation, new_transaction_id def _parse_received_doc_response(self, response): @@ -243,9 +249,9 @@ class HTTPDocFetcher(object): source_replica_uid=self.source_replica_uid) -def _emit_receive_status(received_docs, total): +def _emit_receive_status(uuid, received_docs, total): content = {'received': received_docs, 'total': total} - emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, content) + emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, uuid, content) if received_docs % 20 == 0: msg = "%d/%d" % (received_docs, total) diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index e8abf35b..3fc5a171 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -31,6 +31,12 @@ class HTTPDocSender(object): MAX_BATCH_SIZE = 0 # disabled by now, this is being tested yet + # The uuid of the local replica. + # Any class inheriting from this one should provide a meaningful attribute + # if the sync status event is meant to be used somewhere else. + + uuid = 'undefined' + @defer.inlineCallbacks def _send_docs(self, docs_by_generation, last_known_generation, last_known_trans_id, sync_id): @@ -71,7 +77,8 @@ class HTTPDocSender(object): result = yield self._send_request(body.pop()) if self._defer_encryption: self._delete_sent(sent) - _emit_send_status(body.consumed, total) + + _emit_send_status(self.uuid, body.consumed, total) defer.returnValue(result) def _send_request(self, body): @@ -112,9 +119,9 @@ class HTTPDocSender(object): return d -def _emit_send_status(idx, total): +def _emit_send_status(uuid, idx, total): content = {'sent': idx, 'total': total} - emit_async(SOLEDAD_SYNC_SEND_STATUS, content) + emit_async(SOLEDAD_SYNC_SEND_STATUS, uuid, content) msg = "%d/%d" % (idx, total) logger.debug("Sync send status: %s" % msg) -- cgit v1.2.3 From 402e00bbd3dd98363ce7ffa31cb9aeae1490095e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 7 Dec 2015 12:06:51 -0400 Subject: [feat] use userid in soledad events too for the moment, userid has to be passed to constructor. eventually, we might drop support for passing uuid, since it will be mapped in the service tree --- client/src/leap/soledad/client/api.py | 11 +++++++++-- client/src/leap/soledad/client/http_target/fetch.py | 8 +++++--- client/src/leap/soledad/client/http_target/send.py | 6 ++++-- client/src/leap/soledad/client/secrets.py | 19 +++++++++++++------ 4 files changed, 31 insertions(+), 13 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 0d0d636c..02d9c735 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -125,7 +125,8 @@ class Soledad(object): def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, shared_db=None, - auth_token=None, defer_encryption=False, syncable=True): + auth_token=None, defer_encryption=False, syncable=True, + userid=None): """ Initialize configuration, cryptographic keys and dbs. @@ -179,6 +180,7 @@ class Soledad(object): """ # store config params self._uuid = uuid + self._userid = userid self._passphrase = passphrase self._local_db_path = local_db_path self._server_url = server_url @@ -648,6 +650,10 @@ class Soledad(object): def uuid(self): return self._uuid + @property + def userid(self): + return self._userid + # # ISyncableStorage # @@ -718,8 +724,9 @@ class Soledad(object): return failure def _emit_done_data_sync(passthrough): + user_data = {'uuid': self.uuid, 'userid': self.userid} soledad_events.emit_async( - soledad_events.SOLEDAD_DONE_DATA_SYNC, self.uuid) + soledad_events.SOLEDAD_DONE_DATA_SYNC, user_data) return passthrough d.addCallbacks(_sync_callback, _sync_errback) diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index 087da3a3..9f7a4193 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -44,6 +44,7 @@ class HTTPDocFetcher(object): # if the sync status event is meant to be used somewhere else. uuid = 'undefined' + userid = 'undefined' @defer.inlineCallbacks def _receive_docs(self, last_known_generation, last_known_trans_id, @@ -182,7 +183,8 @@ class HTTPDocFetcher(object): # end of symmetric decryption # ------------------------------------------------------------- self._received_docs += 1 - _emit_receive_status(self.uuid, self._received_docs, total) + user_data = {'uuid': self.uuid, 'userid': self.userid} + _emit_receive_status(user_data, self._received_docs, total) return number_of_changes, new_generation, new_transaction_id def _parse_received_doc_response(self, response): @@ -249,9 +251,9 @@ class HTTPDocFetcher(object): source_replica_uid=self.source_replica_uid) -def _emit_receive_status(uuid, received_docs, total): +def _emit_receive_status(user_data, received_docs, total): content = {'received': received_docs, 'total': total} - emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, uuid, content) + emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, user_data, content) if received_docs % 20 == 0: msg = "%d/%d" % (received_docs, total) diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index 3fc5a171..89288779 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -36,6 +36,7 @@ class HTTPDocSender(object): # if the sync status event is meant to be used somewhere else. uuid = 'undefined' + userid = 'undefined' @defer.inlineCallbacks def _send_docs(self, docs_by_generation, last_known_generation, @@ -78,6 +79,7 @@ class HTTPDocSender(object): if self._defer_encryption: self._delete_sent(sent) + user_data = {'uuid': self.uuid, 'userid': self.userid} _emit_send_status(self.uuid, body.consumed, total) defer.returnValue(result) @@ -119,9 +121,9 @@ class HTTPDocSender(object): return d -def _emit_send_status(uuid, idx, total): +def _emit_send_status(user_data, idx, total): content = {'sent': idx, 'total': total} - emit_async(SOLEDAD_SYNC_SEND_STATUS, uuid, content) + emit_async(SOLEDAD_SYNC_SEND_STATUS, user_data, content) msg = "%d/%d" % (idx, total) logger.debug("Sync send status: %s" % msg) diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index c3c3dff5..90e73bd5 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -147,7 +147,7 @@ class SoledadSecrets(object): Keys used to access storage secrets in recovery documents. """ - def __init__(self, uuid, passphrase, secrets_path, shared_db): + def __init__(self, uuid, passphrase, secrets_path, shared_db, userid=None): """ Initialize the secrets manager. @@ -167,6 +167,7 @@ class SoledadSecrets(object): # param secret_id: The id of the storage secret to be used. self._uuid = uuid + self._userid = userid self._passphrase = passphrase self._secrets_path = secrets_path self._shared_db = shared_db @@ -433,7 +434,8 @@ class SoledadSecrets(object): :return: a document with encrypted key material in its contents :rtype: document.SoledadDocument """ - events.emit_async(events.SOLEDAD_DOWNLOADING_KEYS, self._uuid) + user_data = self._get_user_data() + events.emit_async(events.SOLEDAD_DOWNLOADING_KEYS, user_data) db = self._shared_db if not db: logger.warning('No shared db found') @@ -462,13 +464,14 @@ class SoledadSecrets(object): # fill doc with encrypted secrets doc.content = self._export_recovery_document() # upload secrets to server - events.emit_async(events.SOLEDAD_UPLOADING_KEYS, self._uuid) + user_data = self._get_user_data() + events.emit_async(events.SOLEDAD_UPLOADING_KEYS, user_data) db = self._shared_db if not db: logger.warning('No shared db found') return db.put_doc(doc) - events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, self._uuid) + events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, user_data) # # Management of secret for symmetric encryption. @@ -588,13 +591,14 @@ class SoledadSecrets(object): :return: The id of the generated secret. :rtype: str """ - events.emit_async(events.SOLEDAD_CREATING_KEYS, self._uuid) + user_data = self._get_user_data() + events.emit_async(events.SOLEDAD_CREATING_KEYS, user_data) # generate random secret secret = os.urandom(self.GEN_SECRET_LENGTH) secret_id = sha256(secret).hexdigest() self._secrets[secret_id] = secret self._store_secrets() - events.emit_async(events.SOLEDAD_DONE_CREATING_KEYS, self._uuid) + events.emit_async(events.SOLEDAD_DONE_CREATING_KEYS, user_data) return secret_id def _store_secrets(self): @@ -738,3 +742,6 @@ class SoledadSecrets(object): salt=self._get_sync_db_salt(), buflen=32, # we need a key with 256 bits (32 bytes) ) + + def _get_user_data(self): + return {'uuid': self._uuid, 'userid': self._userid} -- cgit v1.2.3 From a750a80d35b19d1d8bd410c194531b8b92c6eb96 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 14 Dec 2015 10:12:15 -0400 Subject: [bug] fix failing tests after last events modification --- client/src/leap/soledad/client/api.py | 2 +- client/src/leap/soledad/client/secrets.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 02d9c735..619671fd 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -252,7 +252,7 @@ class Soledad(object): """ self._secrets = SoledadSecrets( self.uuid, self._passphrase, self._secrets_path, - self.shared_db) + self.shared_db, userid=self._userid) self._secrets.bootstrap() def _init_u1db_sqlcipher_backend(self): diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 90e73bd5..e2a5a1d7 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -441,7 +441,8 @@ class SoledadSecrets(object): logger.warning('No shared db found') return doc = db.get_doc(self._shared_db_doc_id()) - events.emit_async(events.SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid) + user_data = {'userid': self._userid, 'uuid': self._uuid} + events.emit_async(events.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) return doc def _put_secrets_in_shared_db(self): -- cgit v1.2.3 From 869aa5354ff74be9e0203c37c5d54c79d8e32863 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Nov 2015 22:13:34 -0400 Subject: [feat] set syncable attribute this allows to switch the online/offline mode on a running soledad instance. --- client/src/leap/soledad/client/api.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 619671fd..78f9c652 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -187,6 +187,7 @@ class Soledad(object): self._defer_encryption = defer_encryption self._secrets_path = None self._sync_enc_pool = None + self._dbsyncer = None self.shared_db = shared_db @@ -217,6 +218,7 @@ class Soledad(object): # # initialization/destruction methods # + def _init_config_with_defaults(self): """ Initialize configuration using default values for missing params. @@ -658,6 +660,21 @@ class Soledad(object): # ISyncableStorage # + def set_syncable(self, syncable): + """ + Toggle the syncable state for this database. + + This can be used to start a database with offline state and switch it + online afterwards. Or the opposite: stop syncs when connection is lost. + + :param syncable: new status for syncable. + :type syncable: bool + """ + # TODO should check that we've got a token! + self.shared_db.syncable = syncable + if syncable and not self._dbsyncer: + self._init_u1db_syncer() + def sync(self, defer_decryption=True): """ Synchronize documents with the server replica. @@ -754,6 +771,14 @@ class Soledad(object): """ return self.sync_lock.locked + @property + def syncable(self): + if self.shared_db: + return self.shared_db.syncable + else: + return False + + def _set_token(self, token): """ Set the authentication token for remote database access. -- cgit v1.2.3 From 0d7470b48411f1b9d48762e33533bb9041676c2e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 2 Dec 2015 15:09:58 -0400 Subject: [feat] get_or_create_service_token --- client/src/leap/soledad/client/api.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 78f9c652..1f81f2ab 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -32,6 +32,7 @@ import logging import os import socket import ssl +import uuid import urlparse try: @@ -44,11 +45,11 @@ from StringIO import StringIO from collections import defaultdict from u1db.remote import http_client from u1db.remote.ssl_match_hostname import match_hostname +from twisted.internet.defer import DeferredLock, returnValue, inlineCallbacks from zope.interface import implements from leap.common.config import get_path_prefix from leap.common.plugins import collect_plugins -from twisted.internet.defer import DeferredLock from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import soledad_assert @@ -943,6 +944,38 @@ class Soledad(object): """ return self._dbpool.runOperation(*args, **kw) + # + # Service authentication + # + + @inlineCallbacks + def get_or_create_service_token(self, service): + """ + Return the stored token for a given service, or generates and stores a + random one if it does not exist. + + These tokens can be used to authenticate services. + """ + # FIXME this could use the local sqlcipher database, to avoid + # problems with different replicas creating different tokens. + + yield self.create_index('by-servicetoken', 'type', 'service') + docs = yield self._get_token_for_service(service) + if docs: + doc = docs[0] + returnValue(doc.content['token']) + else: + token = str(uuid.uuid4()).replace('-', '')[-24:] + yield self._set_token_for_service(service, token) + returnValue(token) + + def _get_token_for_service(self, service): + return self.get_from_index('by-servicetoken', 'servicetoken', service) + + def _set_token_for_service(self, service, token): + doc = {'type': 'servicetoken', 'service': service, 'token': token} + return self.create_doc(doc) + def _convert_to_unicode(content): """ -- cgit v1.2.3 From da58f335596c856dee21e2ad014717912e4206d3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 15 Dec 2015 16:44:59 -0400 Subject: [style] pep8 --- client/src/leap/soledad/client/api.py | 1 - 1 file changed, 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 1f81f2ab..e657c939 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -779,7 +779,6 @@ class Soledad(object): else: return False - def _set_token(self, token): """ Set the authentication token for remote database access. -- cgit v1.2.3 From 4c214b7ff9351dd98efbb033dd5e09b8ff9ff763 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 20 Jan 2016 07:54:35 -0300 Subject: [Fix] slow IO-bound calls block reactor - Move them to a thread so reactor can continue processing e.g. http requests --- client/src/leap/soledad/client/encdecpool.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 0954c1df..34667a1e 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -28,6 +28,7 @@ import json import logging from twisted.internet import reactor +from twisted.internet import threads from twisted.internet import defer from twisted.python import log @@ -687,7 +688,11 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ insertable = yield self._get_insertable_docs() for doc_fields in insertable: - self._insert_decrypted_local_doc(*doc_fields) + method = self._insert_decrypted_local_doc + # FIXME: This is used only because SQLCipherU1DBSync is synchronous + # When adbapi is used there is no need for an external thread + # Without this the reactor can freeze and fail docs download + yield threads.deferToThread(method, *doc_fields) defer.returnValue(insertable) def _delete_processed_docs(self, inserted): -- cgit v1.2.3 From 456e6911445dd255580ff023f12e1db3c6a91f7d Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 26 Jan 2016 16:05:20 -0300 Subject: [feat] use cryptography instead of pycryptopp cryptography comes from OpenSSL and Twisted dependencies, so it's already installed. This commit removes a compiled dependency, also possibly making it easier to use on Windows. --- client/src/leap/soledad/client/crypto.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 90ad656e..07a3eaab 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -24,7 +24,8 @@ import hashlib import json import logging -from pycryptopp.cipher.aes import AES +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type @@ -56,7 +57,10 @@ def encrypt_sym(data, key): (len(key) * 8)) iv = os.urandom(16) - ciphertext = AES(key=key, iv=iv).process(data) + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + encryptor = cipher.encryptor() + ciphertext = encryptor.update(data) + encryptor.finalize() return binascii.b2a_base64(iv), ciphertext @@ -81,8 +85,11 @@ def decrypt_sym(data, key, iv): soledad_assert( len(key) == 32, # 32 x 8 = 256 bits. 'Wrong key size: %s (must be 256 bits long).' % len(key)) - return AES( - key=key, iv=binascii.a2b_base64(iv)).process(data) + backend = default_backend() + iv = binascii.a2b_base64(iv) + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + decryptor = cipher.decryptor() + return decryptor.update(data) + decryptor.finalize() def doc_mac_key(doc_id, secret): -- cgit v1.2.3 From 45da47b7b5cddfe40820a8589d82527c2629adae Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 9 Mar 2016 12:50:08 -0400 Subject: [bug] specify openssl backend explicitely for some reason, available_backends does not work inside a frozen PyInstaller binary. - Resolves: #7952 --- client/src/leap/soledad/client/crypto.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 07a3eaab..363d71b9 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -25,7 +25,8 @@ import json import logging from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.backends.multibackend import MultiBackend +from cryptography.hazmat.backends.openssl.backend import Backend as OpenSSLBackend from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type @@ -57,7 +58,7 @@ def encrypt_sym(data, key): (len(key) * 8)) iv = os.urandom(16) - backend = default_backend() + backend = MultiBackend([OpenSSLBackend()]) cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) encryptor = cipher.encryptor() ciphertext = encryptor.update(data) + encryptor.finalize() @@ -85,7 +86,7 @@ def decrypt_sym(data, key, iv): soledad_assert( len(key) == 32, # 32 x 8 = 256 bits. 'Wrong key size: %s (must be 256 bits long).' % len(key)) - backend = default_backend() + backend = MultiBackend([OpenSSLBackend()]) iv = binascii.a2b_base64(iv) cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) decryptor = cipher.decryptor() -- cgit v1.2.3 From b69ba40de308f06ae80f9847acbff40955fdee14 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 1 Apr 2016 17:51:08 -0400 Subject: [pkg] updated to versioneer 0.16 (patched) --- client/src/leap/soledad/client/_version.py | 548 +++++++++++++++++++++-------- 1 file changed, 408 insertions(+), 140 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/_version.py b/client/src/leap/soledad/client/_version.py index 588e4eb5..3ee3f81b 100644 --- a/client/src/leap/soledad/client/_version.py +++ b/client/src/leap/soledad/client/_version.py @@ -1,74 +1,157 @@ + # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) +# versioneer-0.16 (https://github.com/warner/python-versioneer) -# these strings will be replaced by git during git-archive +"""Git implementation of _version.py.""" +import errno +import os +import re import subprocess import sys -import re -import os.path -IN_LONG_VERSION_PY = True -git_refnames = "$Format:%d$" -git_full = "$Format:%H$" +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "src/leap/soledad/client/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: if verbose: - print("unable to run %s" % args[0]) - print(e) + print("unable to find command, tried %s" % (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %s (error)" % args[0]) + print("unable to run %s (error)" % dispcmd) return None return stdout -def get_expanded_variables(versionfile_source): +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source, "r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -84,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) + print("discarding '%s', no digits" % ",".join(refs-tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -94,123 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): if verbose: print("picking %s" % r) return {"version": r, - "full": variables["full"].strip()} - # no suitable tags, so we use the full revision id + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return {"version": variables["full"].strip(), - "full": variables["full"].strip()} - - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname( - os.path.join('..', here)) - ###################################################### - # XXX patch for our specific configuration with - # the three projects leap.soledad.{common, client, server} - # inside the same repo. - ###################################################### - root = os.path.dirname(os.path.join('..', root)) +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.cmd" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % - (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): root = os.path.dirname(root) - else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start " - "with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "" -parentdir_prefix = "leap.soledad.client-" -versionfile_source = "src/leap/soledad/client/_version.py" - - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = {"refnames": git_refnames, "full": git_full} - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} -- cgit v1.2.3