From b9de2af55ab7fbccadf1fdb92d06858c9f29acfa Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 20 Oct 2015 16:50:17 -0300 Subject: [refactor] move couch errors and expose methods errors.py was holding a few specific CouchDB errors, now moved into couch.errors module. Also, some of CouchDatabase methods were declared as private, but external classes needs them. --- client/src/leap/soledad/client/sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index 2276db2a..626ad2e5 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -22,7 +22,7 @@ import logging from twisted.internet import defer from u1db import errors -from leap.soledad.common.errors import MissingDesignDocError +from leap.soledad.common.couch.errors import MissingDesignDocError from u1db.sync import Synchronizer -- cgit v1.2.3 From bb228d13aa14cabe7448f9c7e5b55744de36843a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 19 Nov 2015 18:22:14 -0400 Subject: [bug] fix indirect dependency on couchdb for client by subclassing the MissingDesignDocError, we don't have to import the soledad.common.couch submodule into the soledad.client.sync - Resolves: #7626 --- client/src/leap/soledad/client/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index 626ad2e5..1879031f 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -22,7 +22,7 @@ import logging from twisted.internet import defer from u1db import errors -from leap.soledad.common.couch.errors import MissingDesignDocError +from leap.soledad.common.errors import BackendNotReadyError from u1db.sync import Synchronizer @@ -74,7 +74,7 @@ class SoledadSynchronizer(Synchronizer): (self.target_replica_uid, target_gen, target_trans_id, target_my_gen, target_my_trans_id) = yield \ sync_target.get_sync_info(self.source._replica_uid) - except (errors.DatabaseDoesNotExist, MissingDesignDocError) as e: + except (errors.DatabaseDoesNotExist, BackendNotReadyError) as e: logger.debug("Database isn't ready on server. Will be created.") logger.debug("Reason: %s", e.__class__) self.target_replica_uid = None -- cgit v1.2.3 From 55f45b770a57d1c5f54a66a490aeeea7edae0184 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 1 Dec 2015 15:58:57 -0300 Subject: [bug] fire callback after reseting instance vars If we reset the vars after firing the finish callback, other thread can pick up a dirty state on due concurrency. --- client/src/leap/soledad/client/encdecpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 6d3c11b9..9333578b 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -807,6 +807,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): self._finish() def _finish(self): - self._deferred.callback(None) self._processed_docs = 0 self._last_inserted_idx = 0 + self._deferred.callback(None) -- cgit v1.2.3 From 841e6712ff9ff1ce2b8a5fe92012d89c2aec7077 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 1 Dec 2015 16:06:26 -0300 Subject: [bug] concurrency bug while querying and inserting This line was missing an yield and without it we end up inserting a document that is being retrieved and bad things happen. This is the core fix from yesterday debugging session. During sequential syncs the pool was inserting and querying at the same time and sometimes repeating or failing to delete documents. --- client/src/leap/soledad/client/encdecpool.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 9333578b..a01d3b71 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -70,11 +70,13 @@ class SyncEncryptDecryptPool(object): self._started = False def start(self): + if self.running: + return self._create_pool() self._started = True def stop(self): - if not self._started: + if not self.running: return self._started = False self._destroy_pool() @@ -650,14 +652,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): last_idx = self._last_inserted_idx for doc_id, rev, content, gen, trans_id, encrypted, idx in \ decrypted_docs: - # XXX for some reason, a document might not have been deleted from - # the database. This is a bug. In this point, already - # processed documents should have been removed from the sync - # database and we should not have to skip them here. We need - # to find out why this is happening, fix, and remove the - # skipping below. - if (idx < last_idx + 1): - continue if (idx != last_idx + 1): break insertable.append((doc_id, rev, content, gen, trans_id, idx)) @@ -763,6 +757,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): query = "DELETE FROM %s WHERE 1" % (self.TABLE_NAME,) return self._runOperation(query) + @defer.inlineCallbacks def _collect_async_decryption_results(self): """ Collect the results of the asynchronous doc decryptions and re-raise @@ -773,7 +768,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): async_results = self._async_results[:] for res in async_results: if res.ready(): - self._decrypt_doc_cb(res.get()) # might raise an exception! + yield self._decrypt_doc_cb(res.get()) # might raise an exception! self._async_results.remove(res) @defer.inlineCallbacks @@ -796,7 +791,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): if processed < pending: yield self._async_decrypt_received_docs() - self._collect_async_decryption_results() + yield self._collect_async_decryption_results() docs = yield self._process_decrypted_docs() yield self._delete_processed_docs(docs) # recurse -- cgit v1.2.3 From 3f327c4f472f43d281e3bd7be67aaa9ce3f7d822 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 1 Dec 2015 16:38:02 -0300 Subject: [style] fix pep8 --- client/src/leap/soledad/client/encdecpool.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index a01d3b71..0954c1df 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -768,7 +768,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): async_results = self._async_results[:] for res in async_results: if res.ready(): - yield self._decrypt_doc_cb(res.get()) # might raise an exception! + # XXX: might raise an exception! + yield self._decrypt_doc_cb(res.get()) self._async_results.remove(res) @defer.inlineCallbacks -- cgit v1.2.3 From 6fd543b9fd9679f4978aeedee32eeece5593acc3 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 13 Nov 2015 22:10:18 -0300 Subject: [feat] Adds support to batching limited by size u1db provides batching by default. Current Soledad HTTPS Sync Target was stuck at 1 doc per request. This commit adds batching capability, limiting the size to a predefined value. Default limit size: 500kB --- client/src/leap/soledad/client/http_target/send.py | 46 +++++++++++++++------- .../src/leap/soledad/client/http_target/support.py | 13 ++++-- 2 files changed, 41 insertions(+), 18 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index 80483f0d..c1252c13 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -29,6 +29,8 @@ class HTTPDocSender(object): They need to be encrypted and metadata prepared before sending. """ + MAX_BATCH_SIZE = 500 * 1000 # 500kB by default + @defer.inlineCallbacks def _send_docs(self, docs_by_generation, last_known_generation, last_known_trans_id, sync_id): @@ -43,25 +45,41 @@ class HTTPDocSender(object): sync_id=sync_id, ensure=self._ensure_callback is not None) total = len(docs_by_generation) - for idx, entry in enumerate(docs_by_generation, 1): - yield self._prepare_one_doc(entry, body, idx, total) - result = yield self._http_request( - self._url, - method='POST', - body=body.pop(1), - content_type='application/x-soledad-sync-put') - if self._defer_encryption: - self._delete_sent(idx, docs_by_generation) - _emit_send_status(idx, total) + while body.consumed < total: + result = yield self._send_batch(total, body, docs_by_generation) response_dict = json.loads(result)[0] gen_after_send = response_dict['new_generation'] trans_id_after_send = response_dict['new_transaction_id'] defer.returnValue([gen_after_send, trans_id_after_send]) - def _delete_sent(self, idx, docs_by_generation): - doc = docs_by_generation[idx - 1][0] - self._sync_enc_pool.delete_encrypted_doc( - doc.doc_id, doc.rev) + def _delete_sent(self, docs): + for doc, gen, trans_id in docs: + self._sync_enc_pool.delete_encrypted_doc( + doc.doc_id, doc.rev) + + @defer.inlineCallbacks + def _send_batch(self, total, body, docs): + sent = [] + missing = total - body.consumed + for i in xrange(1, missing + 1): + if body.pending_size > self.MAX_BATCH_SIZE: + break + idx = body.consumed + i + entry = docs[idx - 1] + sent.append(entry) + yield self._prepare_one_doc(entry, body, idx, total) + result = yield self._send_request(body.pop()) + if self._defer_encryption: + self._delete_sent(sent) + _emit_send_status(body.consumed, total) + defer.returnValue(result) + + def _send_request(self, body): + return self._http_request( + self._url, + method='POST', + body=body, + content_type='application/x-soledad-sync-put') @defer.inlineCallbacks def _prepare_one_doc(self, entry, body, idx, total): diff --git a/client/src/leap/soledad/client/http_target/support.py b/client/src/leap/soledad/client/http_target/support.py index 44cd7089..2625744c 100644 --- a/client/src/leap/soledad/client/http_target/support.py +++ b/client/src/leap/soledad/client/http_target/support.py @@ -152,6 +152,8 @@ class RequestBody(object): """ self.headers = header_dict self.entries = [] + self.consumed = 0 + self.pending_size = 0 def insert_info(self, **entry_dict): """ @@ -165,11 +167,11 @@ class RequestBody(object): """ entry = json.dumps(entry_dict) self.entries.append(entry) - return len(entry) + self.pending_size += len(entry) - def pop(self, number=1): + def pop(self): """ - Removes an amount of entries and returns it formatted and ready + Removes all entries and returns it formatted and ready to be sent. :param number: number of entries to pop and format @@ -178,7 +180,10 @@ class RequestBody(object): :return: formatted body ready to be sent :rtype: str """ - entries = [self.entries.pop(0) for i in xrange(number)] + entries = self.entries[:] + self.entries = [] + self.pending_size = 0 + self.consumed += len(entries) return self.entries_to_str(entries) def __str__(self): -- cgit v1.2.3 From 7208d8bc5e5f23d0773533b15763f64d236489b4 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 3 Dec 2015 19:34:56 -0300 Subject: [feat] set default to False on batching for now All batching code has no effect by default with this commit. Since we know that this is a dangerous new feature we will enable them only on our test servers and check them manually before setting it as default or adding more configuration features. Use SyncTarget and server conf file to enable it for testing. --- client/src/leap/soledad/client/http_target/send.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index c1252c13..e8abf35b 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -29,7 +29,7 @@ class HTTPDocSender(object): They need to be encrypted and metadata prepared before sending. """ - MAX_BATCH_SIZE = 500 * 1000 # 500kB by default + MAX_BATCH_SIZE = 0 # disabled by now, this is being tested yet @defer.inlineCallbacks def _send_docs(self, docs_by_generation, last_known_generation, -- cgit v1.2.3 From 2ed8f2b64f3093255d03f282917eb88aac9ab124 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 30 Nov 2015 12:21:56 -0400 Subject: [feat] use uuid in events, multi-user aware - Resolves: #7656 - Releases: 0.8.0 --- client/src/leap/soledad/client/api.py | 6 +++--- client/src/leap/soledad/client/http_target/__init__.py | 1 + client/src/leap/soledad/client/http_target/api.py | 5 +++++ client/src/leap/soledad/client/http_target/fetch.py | 12 +++++++++--- client/src/leap/soledad/client/http_target/send.py | 13 ++++++++++--- 5 files changed, 28 insertions(+), 9 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 8c5f7f1b..0d0d636c 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -102,11 +102,11 @@ class Soledad(object): soledad starts to retrieve keys from server. SOLEDAD_DONE_DOWNLOADING_KEYS: emitted during bootstrap sequence when soledad finishes downloading keys from server. - SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when - there's indeed new data to be synchronized between local database - replica and server's replica. SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has finished synchronizing with remote replica. + SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when + there's indeed new data to be synchronized between local database + replica and server's replica. --- not used right now. """ implements(soledad_interfaces.ILocalStorage, soledad_interfaces.ISyncableStorage, diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py index 498fb6e7..a16531ef 100644 --- a/client/src/leap/soledad/client/http_target/__init__.py +++ b/client/src/leap/soledad/client/http_target/__init__.py @@ -79,6 +79,7 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): self._url = str(url) + "/sync-from/" + str(source_replica_uid) self.source_replica_uid = source_replica_uid self._auth_header = None + self._uuid = None self.set_creds(creds) self._crypto = crypto self._sync_db = sync_db diff --git a/client/src/leap/soledad/client/http_target/api.py b/client/src/leap/soledad/client/http_target/api.py index dcc762f6..94354092 100644 --- a/client/src/leap/soledad/client/http_target/api.py +++ b/client/src/leap/soledad/client/http_target/api.py @@ -40,6 +40,10 @@ class SyncTargetAPI(SyncTarget): self._sync_decr_pool.stop() yield self._http.close() + @property + def uuid(self): + return self._uuid + def set_creds(self, creds): """ Update credentials. @@ -49,6 +53,7 @@ class SyncTargetAPI(SyncTarget): """ uuid = creds['token']['uuid'] token = creds['token']['token'] + self._uuid = uuid auth = '%s:%s' % (uuid, token) b64_token = base64.b64encode(auth) self._auth_header = {'Authorization': ['Token %s' % b64_token]} diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index 65e576d9..087da3a3 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -39,6 +39,12 @@ class HTTPDocFetcher(object): So we parse, decrypt and insert locally as they arrive. """ + # The uuid of the local replica. + # Any class inheriting from this one should provide a meaningful attribute + # if the sync status event is meant to be used somewhere else. + + uuid = 'undefined' + @defer.inlineCallbacks def _receive_docs(self, last_known_generation, last_known_trans_id, ensure_callback, sync_id, defer_decryption): @@ -176,7 +182,7 @@ class HTTPDocFetcher(object): # end of symmetric decryption # ------------------------------------------------------------- self._received_docs += 1 - _emit_receive_status(self._received_docs, total) + _emit_receive_status(self.uuid, self._received_docs, total) return number_of_changes, new_generation, new_transaction_id def _parse_received_doc_response(self, response): @@ -243,9 +249,9 @@ class HTTPDocFetcher(object): source_replica_uid=self.source_replica_uid) -def _emit_receive_status(received_docs, total): +def _emit_receive_status(uuid, received_docs, total): content = {'received': received_docs, 'total': total} - emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, content) + emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, uuid, content) if received_docs % 20 == 0: msg = "%d/%d" % (received_docs, total) diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index e8abf35b..3fc5a171 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -31,6 +31,12 @@ class HTTPDocSender(object): MAX_BATCH_SIZE = 0 # disabled by now, this is being tested yet + # The uuid of the local replica. + # Any class inheriting from this one should provide a meaningful attribute + # if the sync status event is meant to be used somewhere else. + + uuid = 'undefined' + @defer.inlineCallbacks def _send_docs(self, docs_by_generation, last_known_generation, last_known_trans_id, sync_id): @@ -71,7 +77,8 @@ class HTTPDocSender(object): result = yield self._send_request(body.pop()) if self._defer_encryption: self._delete_sent(sent) - _emit_send_status(body.consumed, total) + + _emit_send_status(self.uuid, body.consumed, total) defer.returnValue(result) def _send_request(self, body): @@ -112,9 +119,9 @@ class HTTPDocSender(object): return d -def _emit_send_status(idx, total): +def _emit_send_status(uuid, idx, total): content = {'sent': idx, 'total': total} - emit_async(SOLEDAD_SYNC_SEND_STATUS, content) + emit_async(SOLEDAD_SYNC_SEND_STATUS, uuid, content) msg = "%d/%d" % (idx, total) logger.debug("Sync send status: %s" % msg) -- cgit v1.2.3 From 402e00bbd3dd98363ce7ffa31cb9aeae1490095e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 7 Dec 2015 12:06:51 -0400 Subject: [feat] use userid in soledad events too for the moment, userid has to be passed to constructor. eventually, we might drop support for passing uuid, since it will be mapped in the service tree --- client/src/leap/soledad/client/api.py | 11 +++++++++-- client/src/leap/soledad/client/http_target/fetch.py | 8 +++++--- client/src/leap/soledad/client/http_target/send.py | 6 ++++-- client/src/leap/soledad/client/secrets.py | 19 +++++++++++++------ 4 files changed, 31 insertions(+), 13 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 0d0d636c..02d9c735 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -125,7 +125,8 @@ class Soledad(object): def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, shared_db=None, - auth_token=None, defer_encryption=False, syncable=True): + auth_token=None, defer_encryption=False, syncable=True, + userid=None): """ Initialize configuration, cryptographic keys and dbs. @@ -179,6 +180,7 @@ class Soledad(object): """ # store config params self._uuid = uuid + self._userid = userid self._passphrase = passphrase self._local_db_path = local_db_path self._server_url = server_url @@ -648,6 +650,10 @@ class Soledad(object): def uuid(self): return self._uuid + @property + def userid(self): + return self._userid + # # ISyncableStorage # @@ -718,8 +724,9 @@ class Soledad(object): return failure def _emit_done_data_sync(passthrough): + user_data = {'uuid': self.uuid, 'userid': self.userid} soledad_events.emit_async( - soledad_events.SOLEDAD_DONE_DATA_SYNC, self.uuid) + soledad_events.SOLEDAD_DONE_DATA_SYNC, user_data) return passthrough d.addCallbacks(_sync_callback, _sync_errback) diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index 087da3a3..9f7a4193 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -44,6 +44,7 @@ class HTTPDocFetcher(object): # if the sync status event is meant to be used somewhere else. uuid = 'undefined' + userid = 'undefined' @defer.inlineCallbacks def _receive_docs(self, last_known_generation, last_known_trans_id, @@ -182,7 +183,8 @@ class HTTPDocFetcher(object): # end of symmetric decryption # ------------------------------------------------------------- self._received_docs += 1 - _emit_receive_status(self.uuid, self._received_docs, total) + user_data = {'uuid': self.uuid, 'userid': self.userid} + _emit_receive_status(user_data, self._received_docs, total) return number_of_changes, new_generation, new_transaction_id def _parse_received_doc_response(self, response): @@ -249,9 +251,9 @@ class HTTPDocFetcher(object): source_replica_uid=self.source_replica_uid) -def _emit_receive_status(uuid, received_docs, total): +def _emit_receive_status(user_data, received_docs, total): content = {'received': received_docs, 'total': total} - emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, uuid, content) + emit_async(SOLEDAD_SYNC_RECEIVE_STATUS, user_data, content) if received_docs % 20 == 0: msg = "%d/%d" % (received_docs, total) diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index 3fc5a171..89288779 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -36,6 +36,7 @@ class HTTPDocSender(object): # if the sync status event is meant to be used somewhere else. uuid = 'undefined' + userid = 'undefined' @defer.inlineCallbacks def _send_docs(self, docs_by_generation, last_known_generation, @@ -78,6 +79,7 @@ class HTTPDocSender(object): if self._defer_encryption: self._delete_sent(sent) + user_data = {'uuid': self.uuid, 'userid': self.userid} _emit_send_status(self.uuid, body.consumed, total) defer.returnValue(result) @@ -119,9 +121,9 @@ class HTTPDocSender(object): return d -def _emit_send_status(uuid, idx, total): +def _emit_send_status(user_data, idx, total): content = {'sent': idx, 'total': total} - emit_async(SOLEDAD_SYNC_SEND_STATUS, uuid, content) + emit_async(SOLEDAD_SYNC_SEND_STATUS, user_data, content) msg = "%d/%d" % (idx, total) logger.debug("Sync send status: %s" % msg) diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index c3c3dff5..90e73bd5 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -147,7 +147,7 @@ class SoledadSecrets(object): Keys used to access storage secrets in recovery documents. """ - def __init__(self, uuid, passphrase, secrets_path, shared_db): + def __init__(self, uuid, passphrase, secrets_path, shared_db, userid=None): """ Initialize the secrets manager. @@ -167,6 +167,7 @@ class SoledadSecrets(object): # param secret_id: The id of the storage secret to be used. self._uuid = uuid + self._userid = userid self._passphrase = passphrase self._secrets_path = secrets_path self._shared_db = shared_db @@ -433,7 +434,8 @@ class SoledadSecrets(object): :return: a document with encrypted key material in its contents :rtype: document.SoledadDocument """ - events.emit_async(events.SOLEDAD_DOWNLOADING_KEYS, self._uuid) + user_data = self._get_user_data() + events.emit_async(events.SOLEDAD_DOWNLOADING_KEYS, user_data) db = self._shared_db if not db: logger.warning('No shared db found') @@ -462,13 +464,14 @@ class SoledadSecrets(object): # fill doc with encrypted secrets doc.content = self._export_recovery_document() # upload secrets to server - events.emit_async(events.SOLEDAD_UPLOADING_KEYS, self._uuid) + user_data = self._get_user_data() + events.emit_async(events.SOLEDAD_UPLOADING_KEYS, user_data) db = self._shared_db if not db: logger.warning('No shared db found') return db.put_doc(doc) - events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, self._uuid) + events.emit_async(events.SOLEDAD_DONE_UPLOADING_KEYS, user_data) # # Management of secret for symmetric encryption. @@ -588,13 +591,14 @@ class SoledadSecrets(object): :return: The id of the generated secret. :rtype: str """ - events.emit_async(events.SOLEDAD_CREATING_KEYS, self._uuid) + user_data = self._get_user_data() + events.emit_async(events.SOLEDAD_CREATING_KEYS, user_data) # generate random secret secret = os.urandom(self.GEN_SECRET_LENGTH) secret_id = sha256(secret).hexdigest() self._secrets[secret_id] = secret self._store_secrets() - events.emit_async(events.SOLEDAD_DONE_CREATING_KEYS, self._uuid) + events.emit_async(events.SOLEDAD_DONE_CREATING_KEYS, user_data) return secret_id def _store_secrets(self): @@ -738,3 +742,6 @@ class SoledadSecrets(object): salt=self._get_sync_db_salt(), buflen=32, # we need a key with 256 bits (32 bytes) ) + + def _get_user_data(self): + return {'uuid': self._uuid, 'userid': self._userid} -- cgit v1.2.3 From a750a80d35b19d1d8bd410c194531b8b92c6eb96 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 14 Dec 2015 10:12:15 -0400 Subject: [bug] fix failing tests after last events modification --- client/src/leap/soledad/client/api.py | 2 +- client/src/leap/soledad/client/secrets.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 02d9c735..619671fd 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -252,7 +252,7 @@ class Soledad(object): """ self._secrets = SoledadSecrets( self.uuid, self._passphrase, self._secrets_path, - self.shared_db) + self.shared_db, userid=self._userid) self._secrets.bootstrap() def _init_u1db_sqlcipher_backend(self): diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 90e73bd5..e2a5a1d7 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -441,7 +441,8 @@ class SoledadSecrets(object): logger.warning('No shared db found') return doc = db.get_doc(self._shared_db_doc_id()) - events.emit_async(events.SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid) + user_data = {'userid': self._userid, 'uuid': self._uuid} + events.emit_async(events.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data) return doc def _put_secrets_in_shared_db(self): -- cgit v1.2.3 From 869aa5354ff74be9e0203c37c5d54c79d8e32863 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 26 Nov 2015 22:13:34 -0400 Subject: [feat] set syncable attribute this allows to switch the online/offline mode on a running soledad instance. --- client/src/leap/soledad/client/api.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 619671fd..78f9c652 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -187,6 +187,7 @@ class Soledad(object): self._defer_encryption = defer_encryption self._secrets_path = None self._sync_enc_pool = None + self._dbsyncer = None self.shared_db = shared_db @@ -217,6 +218,7 @@ class Soledad(object): # # initialization/destruction methods # + def _init_config_with_defaults(self): """ Initialize configuration using default values for missing params. @@ -658,6 +660,21 @@ class Soledad(object): # ISyncableStorage # + def set_syncable(self, syncable): + """ + Toggle the syncable state for this database. + + This can be used to start a database with offline state and switch it + online afterwards. Or the opposite: stop syncs when connection is lost. + + :param syncable: new status for syncable. + :type syncable: bool + """ + # TODO should check that we've got a token! + self.shared_db.syncable = syncable + if syncable and not self._dbsyncer: + self._init_u1db_syncer() + def sync(self, defer_decryption=True): """ Synchronize documents with the server replica. @@ -754,6 +771,14 @@ class Soledad(object): """ return self.sync_lock.locked + @property + def syncable(self): + if self.shared_db: + return self.shared_db.syncable + else: + return False + + def _set_token(self, token): """ Set the authentication token for remote database access. -- cgit v1.2.3 From 0d7470b48411f1b9d48762e33533bb9041676c2e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 2 Dec 2015 15:09:58 -0400 Subject: [feat] get_or_create_service_token --- client/src/leap/soledad/client/api.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 78f9c652..1f81f2ab 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -32,6 +32,7 @@ import logging import os import socket import ssl +import uuid import urlparse try: @@ -44,11 +45,11 @@ from StringIO import StringIO from collections import defaultdict from u1db.remote import http_client from u1db.remote.ssl_match_hostname import match_hostname +from twisted.internet.defer import DeferredLock, returnValue, inlineCallbacks from zope.interface import implements from leap.common.config import get_path_prefix from leap.common.plugins import collect_plugins -from twisted.internet.defer import DeferredLock from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import soledad_assert @@ -943,6 +944,38 @@ class Soledad(object): """ return self._dbpool.runOperation(*args, **kw) + # + # Service authentication + # + + @inlineCallbacks + def get_or_create_service_token(self, service): + """ + Return the stored token for a given service, or generates and stores a + random one if it does not exist. + + These tokens can be used to authenticate services. + """ + # FIXME this could use the local sqlcipher database, to avoid + # problems with different replicas creating different tokens. + + yield self.create_index('by-servicetoken', 'type', 'service') + docs = yield self._get_token_for_service(service) + if docs: + doc = docs[0] + returnValue(doc.content['token']) + else: + token = str(uuid.uuid4()).replace('-', '')[-24:] + yield self._set_token_for_service(service, token) + returnValue(token) + + def _get_token_for_service(self, service): + return self.get_from_index('by-servicetoken', 'servicetoken', service) + + def _set_token_for_service(self, service, token): + doc = {'type': 'servicetoken', 'service': service, 'token': token} + return self.create_doc(doc) + def _convert_to_unicode(content): """ -- cgit v1.2.3 From da58f335596c856dee21e2ad014717912e4206d3 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 15 Dec 2015 16:44:59 -0400 Subject: [style] pep8 --- client/src/leap/soledad/client/api.py | 1 - 1 file changed, 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 1f81f2ab..e657c939 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -779,7 +779,6 @@ class Soledad(object): else: return False - def _set_token(self, token): """ Set the authentication token for remote database access. -- cgit v1.2.3 From 4c214b7ff9351dd98efbb033dd5e09b8ff9ff763 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 20 Jan 2016 07:54:35 -0300 Subject: [Fix] slow IO-bound calls block reactor - Move them to a thread so reactor can continue processing e.g. http requests --- client/src/leap/soledad/client/encdecpool.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 0954c1df..34667a1e 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -28,6 +28,7 @@ import json import logging from twisted.internet import reactor +from twisted.internet import threads from twisted.internet import defer from twisted.python import log @@ -687,7 +688,11 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ insertable = yield self._get_insertable_docs() for doc_fields in insertable: - self._insert_decrypted_local_doc(*doc_fields) + method = self._insert_decrypted_local_doc + # FIXME: This is used only because SQLCipherU1DBSync is synchronous + # When adbapi is used there is no need for an external thread + # Without this the reactor can freeze and fail docs download + yield threads.deferToThread(method, *doc_fields) defer.returnValue(insertable) def _delete_processed_docs(self, inserted): -- cgit v1.2.3 From 456e6911445dd255580ff023f12e1db3c6a91f7d Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 26 Jan 2016 16:05:20 -0300 Subject: [feat] use cryptography instead of pycryptopp cryptography comes from OpenSSL and Twisted dependencies, so it's already installed. This commit removes a compiled dependency, also possibly making it easier to use on Windows. --- client/src/leap/soledad/client/crypto.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 90ad656e..07a3eaab 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -24,7 +24,8 @@ import hashlib import json import logging -from pycryptopp.cipher.aes import AES +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.backends import default_backend from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type @@ -56,7 +57,10 @@ def encrypt_sym(data, key): (len(key) * 8)) iv = os.urandom(16) - ciphertext = AES(key=key, iv=iv).process(data) + backend = default_backend() + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + encryptor = cipher.encryptor() + ciphertext = encryptor.update(data) + encryptor.finalize() return binascii.b2a_base64(iv), ciphertext @@ -81,8 +85,11 @@ def decrypt_sym(data, key, iv): soledad_assert( len(key) == 32, # 32 x 8 = 256 bits. 'Wrong key size: %s (must be 256 bits long).' % len(key)) - return AES( - key=key, iv=binascii.a2b_base64(iv)).process(data) + backend = default_backend() + iv = binascii.a2b_base64(iv) + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + decryptor = cipher.decryptor() + return decryptor.update(data) + decryptor.finalize() def doc_mac_key(doc_id, secret): -- cgit v1.2.3 From 45da47b7b5cddfe40820a8589d82527c2629adae Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 9 Mar 2016 12:50:08 -0400 Subject: [bug] specify openssl backend explicitely for some reason, available_backends does not work inside a frozen PyInstaller binary. - Resolves: #7952 --- client/src/leap/soledad/client/crypto.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 07a3eaab..363d71b9 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -25,7 +25,8 @@ import json import logging from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.backends.multibackend import MultiBackend +from cryptography.hazmat.backends.openssl.backend import Backend as OpenSSLBackend from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type @@ -57,7 +58,7 @@ def encrypt_sym(data, key): (len(key) * 8)) iv = os.urandom(16) - backend = default_backend() + backend = MultiBackend([OpenSSLBackend()]) cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) encryptor = cipher.encryptor() ciphertext = encryptor.update(data) + encryptor.finalize() @@ -85,7 +86,7 @@ def decrypt_sym(data, key, iv): soledad_assert( len(key) == 32, # 32 x 8 = 256 bits. 'Wrong key size: %s (must be 256 bits long).' % len(key)) - backend = default_backend() + backend = MultiBackend([OpenSSLBackend()]) iv = binascii.a2b_base64(iv) cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) decryptor = cipher.decryptor() -- cgit v1.2.3 From b69ba40de308f06ae80f9847acbff40955fdee14 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 1 Apr 2016 17:51:08 -0400 Subject: [pkg] updated to versioneer 0.16 (patched) --- client/src/leap/soledad/client/_version.py | 548 +++++++++++++++++++++-------- 1 file changed, 408 insertions(+), 140 deletions(-) (limited to 'client/src') diff --git a/client/src/leap/soledad/client/_version.py b/client/src/leap/soledad/client/_version.py index 588e4eb5..3ee3f81b 100644 --- a/client/src/leap/soledad/client/_version.py +++ b/client/src/leap/soledad/client/_version.py @@ -1,74 +1,157 @@ + # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.7+ (https://github.com/warner/python-versioneer) +# versioneer-0.16 (https://github.com/warner/python-versioneer) -# these strings will be replaced by git during git-archive +"""Git implementation of _version.py.""" +import errno +import os +import re import subprocess import sys -import re -import os.path -IN_LONG_VERSION_PY = True -git_refnames = "$Format:%d$" -git_full = "$Format:%H$" +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords -def run_command(args, cwd=None, verbose=False): - try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd) - except EnvironmentError: - e = sys.exc_info()[1] +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "None" + cfg.versionfile_source = "src/leap/soledad/client/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: if verbose: - print("unable to run %s" % args[0]) - print(e) + print("unable to find command, tried %s" % (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %s (error)" % args[0]) + print("unable to run %s (error)" % dispcmd) return None return stdout -def get_expanded_variables(versionfile_source): +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_source, "r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -84,7 +167,7 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): # "stabilization", as well as "HEAD" and "master". tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) + print("discarding '%s', no digits" % ",".join(refs-tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -94,123 +177,308 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): if verbose: print("picking %s" % r) return {"version": r, - "full": variables["full"].strip()} - # no suitable tags, so we use the full revision id + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return {"version": variables["full"].strip(), - "full": variables["full"].strip()} - - -def versions_from_vcs(tag_prefix, versionfile_source, verbose=False): - # this runs 'git' from the root of the source tree. That either means - # someone ran a setup.py command (and this code is in versioneer.py, so - # IN_LONG_VERSION_PY=False, thus the containing directory is the root of - # the source tree), or someone ran a project-specific entry point (and - # this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the - # containing directory is somewhere deeper in the source tree). This only - # gets called if the git-archive 'subst' variables were *not* expanded, - # and _version.py hasn't already been rewritten with a short version - # string, meaning we're inside a checked out source tree. + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} - try: - here = os.path.abspath(__file__) - except NameError: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - return {} # not always correct - - # versionfile_source is the relative path from the top of the source tree - # (where the .git directory might live) to this file. Invert this to find - # the root from __file__. - root = here - if IN_LONG_VERSION_PY: - for i in range(len(versionfile_source.split("/"))): - root = os.path.dirname(root) - else: - root = os.path.dirname( - os.path.join('..', here)) - ###################################################### - # XXX patch for our specific configuration with - # the three projects leap.soledad.{common, client, server} - # inside the same repo. - ###################################################### - root = os.path.dirname(os.path.join('..', root)) +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ if not os.path.exists(os.path.join(root, ".git")): if verbose: print("no .git in %s" % root) - return {} + raise NotThisMethod("no .git directory") - GIT = "git" + GITS = ["git"] if sys.platform == "win32": - GIT = "git.cmd" - stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % - (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - return {"version": tag, "full": full} - - -def versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose=False): - if IN_LONG_VERSION_PY: - # We're running from _version.py. If it's from a source tree - # (execute-in-place), we can work upwards to find the root of the - # tree, and then check the parent directory for a version string. If - # it's in an installed application, there's no hope. - try: - here = os.path.abspath(__file__) - except NameError: - # py2exe/bbfreeze/non-CPython don't have __file__ - return {} # without __file__, we have no hope + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source - # tree to _version.py. Invert this to find the root from __file__. - root = here - for i in range(len(versionfile_source.split("/"))): + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): root = os.path.dirname(root) - else: - # we're running from versioneer.py, which means we're running from - # the setup.py in a source tree. sys.argv[0] is setup.py in the root. - here = os.path.abspath(sys.argv[0]) - root = os.path.dirname(here) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start " - "with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": ""} - -tag_prefix = "" -parentdir_prefix = "leap.soledad.client-" -versionfile_source = "src/leap/soledad/client/_version.py" - - -def get_versions(default={"version": "unknown", "full": ""}, verbose=False): - variables = {"refnames": git_refnames, "full": git_full} - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if not ver: - ver = versions_from_vcs(tag_prefix, versionfile_source, verbose) - if not ver: - ver = versions_from_parentdir(parentdir_prefix, versionfile_source, - verbose) - if not ver: - ver = default - return ver + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} -- cgit v1.2.3