From c8bda65e0029999e1c2587b74d490aee2d05137e Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 14 Jul 2014 19:22:44 -0300 Subject: Add changes file (#5895). --- client/changes/feature_5895-store-all-incoming-documents-in-sync-db | 1 + 1 file changed, 1 insertion(+) create mode 100644 client/changes/feature_5895-store-all-incoming-documents-in-sync-db diff --git a/client/changes/feature_5895-store-all-incoming-documents-in-sync-db b/client/changes/feature_5895-store-all-incoming-documents-in-sync-db new file mode 100644 index 00000000..71d5a91f --- /dev/null +++ b/client/changes/feature_5895-store-all-incoming-documents-in-sync-db @@ -0,0 +1 @@ + o Store all incoming documents in the sync db (#5895). -- cgit v1.2.3 From 6183ad313298de08e05c31c5f18f133361cd803b Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 14 Jul 2014 19:54:43 -0300 Subject: Make client db access script defer decryption. --- scripts/db_access/client_side_db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 6c456c41..1c4f3754 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -26,7 +26,7 @@ from util import ValidateUserHandle # create a logger logger = logging.getLogger(__name__) LOG_FORMAT = '%(asctime)s %(message)s' -logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) +logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) safe_unhexlify = lambda x: binascii.unhexlify(x) if ( @@ -119,7 +119,8 @@ def get_soledad_instance(username, provider, passphrase, basedir): local_db_path=local_db_path, server_url=server_url, cert_file=cert_file, - auth_token=token) + auth_token=token, + defer_encryption=True) # main program -- cgit v1.2.3 From 69f5087c718cc534a969fcba0fcb35812c88ad8b Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 14 Jul 2014 20:01:01 -0300 Subject: Add encrypted field to sync db (#5895). --- client/src/leap/soledad/client/crypto.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 7133f804..89220860 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -691,7 +691,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ # TODO implement throttling to reduce cpu usage?? TABLE_NAME = "docs_received" - FIELD_NAMES = "doc_id, rev, content, gen, trans_id" + FIELD_NAMES = "doc_id, rev, content, gen, trans_id, encrypted" write_encrypted_lock = threading.Lock() @@ -733,13 +733,15 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :type trans_id: str """ docstr = json.dumps(content) - sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?)" % ( + sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % ( self.TABLE_NAME,) con = self._sync_db with self._sync_db_write_lock: with con: - con.execute(sql_ins, (doc_id, doc_rev, docstr, gen, trans_id)) + con.execute( + sql_ins, + (doc_id, doc_rev, docstr, gen, trans_id, 1)) def insert_marker_for_received_doc(self, doc_id, doc_rev, gen): """ @@ -757,12 +759,12 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :param gen: the Document Generation :type gen: int """ - sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?)" % ( + sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % ( self.TABLE_NAME,) con = self._sync_db with self._sync_db_write_lock: with con: - con.execute(sql_ins, (doc_id, doc_rev, '', gen, '')) + con.execute(sql_ins, (doc_id, doc_rev, '', gen, '', 0)) def insert_received_doc(self, doc_id, doc_rev, content, gen, trans_id): """ -- cgit v1.2.3 From 95f34ccab21e36ea48e0d01a4b9ee00e6094d1ec Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 14 Jul 2014 20:05:00 -0300 Subject: Store non-encrypted docs in the sync db (#5895). --- client/src/leap/soledad/client/crypto.py | 40 +++++++------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 89220860..128e40d7 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -743,29 +743,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): sql_ins, (doc_id, doc_rev, docstr, gen, trans_id, 1)) - def insert_marker_for_received_doc(self, doc_id, doc_rev, gen): - """ - Insert a marker with the document id, revision and generation on the - sync db. This document does not have an encrypted payload, so the - content has already been inserted into the decrypted_docs dictionary - from where it can be picked following generation order. - We need to leave here the marker to be able to calculate the expected - insertion order for a synchronization batch. - - :param doc_id: The Document ID. - :type doc_id: str - :param doc_rev: The Document Revision - :param doc_rev: str - :param gen: the Document Generation - :type gen: int - """ - sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % ( - self.TABLE_NAME,) - con = self._sync_db - with self._sync_db_write_lock: - with con: - con.execute(sql_ins, (doc_id, doc_rev, '', gen, '', 0)) - def insert_received_doc(self, doc_id, doc_rev, content, gen, trans_id): """ Insert a document that is not symmetrically encrypted. @@ -783,16 +760,15 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :param trans_id: Transaction ID :type trans_id: str """ - # XXX this need a deeper review / testing. - # I believe that what I'm doing here is prone to problems - # if the sync is interrupted (ie, client crash) in the worst possible - # moment. We would need a recover strategy in that case - # (or, insert the document in the table all the same, but with a flag - # saying if the document is sym-encrypted or not), content = json.dumps(content) - result = doc_id, doc_rev, content, gen, trans_id - self.decrypted_docs[gen] = result - self.insert_marker_for_received_doc(doc_id, doc_rev, gen) + sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % ( + self.TABLE_NAME,) + con = self._sync_db + with self._sync_db_write_lock: + with con: + con.execute( + sql_ins, + (doc_id, doc_rev, content, gen, trans_id, 0)) def delete_encrypted_received_doc(self, doc_id, doc_rev): """ -- cgit v1.2.3 From 51e0bf7f79a444661b10fe418af85b0a60f41afb Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 14 Jul 2014 20:09:01 -0300 Subject: Insert received docs in sync db after decryption (#5895). --- client/src/leap/soledad/client/crypto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 128e40d7..d0a5a693 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -876,7 +876,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ doc_id, rev, content, gen, trans_id = result logger.debug("Sync decrypter pool: decrypted doc %s: %s %s" % (doc_id, rev, gen)) - self.decrypted_docs[gen] = result + self.insert_received_doc(doc_id, rev, content, gen, trans_id) def get_docs_by_generation(self): """ -- cgit v1.2.3 From 54a69eb14189e06556af15dcdf5d5ed424778fc2 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 15 Jul 2014 13:46:42 -0300 Subject: Store all received docs in sync db (#5895). --- client/src/leap/soledad/client/crypto.py | 156 ++++++++++++++++--------------- client/src/leap/soledad/client/target.py | 12 ++- 2 files changed, 89 insertions(+), 79 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index d0a5a693..4a73a910 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -256,11 +256,11 @@ class SoledadCrypto(object): secret = property( _get_secret, doc='The secret used for symmetric encryption') + # # Crypto utilities for a SoledadDocument. # - def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret): """ Calculate a MAC for C{doc} using C{ciphertext}. @@ -657,26 +657,6 @@ def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret): return doc_id, doc_rev, decrypted_content, gen, trans_id -def get_insertable_docs_by_gen(expected, got): - """ - Return a list of documents ready to be inserted. This list is computed - by aligning the expected list with the already gotten docs, and returning - the maximum number of docs that can be processed in the expected order - before finding a gap. - - :param expected: A list of generations to be inserted. - :type expected: list - - :param got: A dictionary whose values are the docs to be inserted. - :type got: dict - """ - ordered = [got.get(i) for i in expected] - if None in ordered: - return ordered[:ordered.index(None)] - else: - return ordered - - class SyncDecrypterPool(SyncEncryptDecryptPool): """ Pool of workers that spawn subprocesses to execute the symmetric decryption @@ -700,10 +680,18 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): Initialize the decrypter pool, and setup a dict for putting the results of the decrypted docs until they are picked by the insert routine that gets them in order. + + :param insert_doc_cb: A callback for inserting received documents from + target. If not overriden, this will call u1db + insert_doc_from_target in synchronizer, which + implements the TAKE OTHER semantics. + :type insert_doc_cb: function + :param last_known_generation: Target's last known generation. + :type last_known_generation: int """ self._insert_doc_cb = kwargs.pop("insert_doc_cb") + self._last_known_generation = kwargs.pop("last_known_generation") SyncEncryptDecryptPool.__init__(self, *args, **kwargs) - self.decrypted_docs = {} self.source_replica_uid = None def set_source_replica_uid(self, source_replica_uid): @@ -733,12 +721,14 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :type trans_id: str """ docstr = json.dumps(content) + sql_del = "DELETE FROM '%s' WHERE doc_id=?" % (self.TABLE_NAME,) sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % ( self.TABLE_NAME,) con = self._sync_db with self._sync_db_write_lock: with con: + con.execute(sql_del, (doc_id, )) con.execute( sql_ins, (doc_id, doc_rev, docstr, gen, trans_id, 1)) @@ -760,20 +750,23 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :param trans_id: Transaction ID :type trans_id: str """ - content = json.dumps(content) + if not isinstance(content, str): + content = json.dumps(content) + sql_del = "DELETE FROM '%s' WHERE doc_id=?" % ( + self.TABLE_NAME,) sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" % ( self.TABLE_NAME,) con = self._sync_db with self._sync_db_write_lock: with con: + con.execute(sql_del, (doc_id,)) con.execute( sql_ins, (doc_id, doc_rev, content, gen, trans_id, 0)) - def delete_encrypted_received_doc(self, doc_id, doc_rev): + def delete_received_doc(self, doc_id, doc_rev): """ - Delete a encrypted received doc after it was inserted into the local - db. + Delete a received doc after it was inserted into the local db. :param doc_id: Document ID. :type doc_id: str @@ -787,7 +780,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): with con: con.execute(sql_del, (doc_id, doc_rev)) - def decrypt_doc(self, doc_id, rev, source_replica_uid, workers=True): + def decrypt_doc(self, doc_id, rev, content, gen, trans_id, + source_replica_uid, workers=True): """ Symmetrically decrypt a document. @@ -795,6 +789,14 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :type doc: str :param rev: The revision of the document. :type rev: str + :param content: The serialized content of the document. + :type content: str + :param gen: The generation corresponding to the modification of that + document. + :type gen: int + :param trans_id: The transaction id corresponding to the modification + of that document. + :type trans_id: str :param source_replica_uid: :type source_replica_uid: str @@ -813,33 +815,14 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): logger.debug("Sync decrypter pool: no insert_doc_cb() yet.") return - # XXX move to get_doc function... - c = self._sync_db.cursor() - sql = "SELECT * FROM '%s' WHERE doc_id=? AND rev=?" % ( - self.TABLE_NAME,) - try: - c.execute(sql, (doc_id, rev)) - res = c.fetchone() - except Exception as exc: - logger.warning("Error getting docs from syncdb: %r" % (exc,)) - return - if res is None: - logger.debug("Doc %s:%s does not exist in sync db" % (doc_id, rev)) - return - soledad_assert(self._crypto is not None, "need a crypto object") - try: - doc_id, rev, docstr, gen, trans_id = res - except ValueError: - logger.warning("Wrong entry in sync db") - return - if len(docstr) == 0: + if len(content) == 0: # not encrypted payload return try: - content = json.loads(docstr) + content = json.loads(content) except TypeError: logger.warning("Wrong type while decoding json: %s" % repr(docstr)) return @@ -867,34 +850,61 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): def decrypt_doc_cb(self, result): """ - Temporarily store the decryption result in a dictionary where it will - be picked by process_decrypted. + Store the decryption result in the sync db from where it will later be + picked by process_decrypted. :param result: A tuple containing the doc id, revision and encrypted content. :type result: tuple(str, str, str) """ doc_id, rev, content, gen, trans_id = result - logger.debug("Sync decrypter pool: decrypted doc %s: %s %s" % (doc_id, rev, gen)) + logger.debug("Sync decrypter pool: decrypted doc %s: %s %s" + % (doc_id, rev, gen)) self.insert_received_doc(doc_id, rev, content, gen, trans_id) - def get_docs_by_generation(self): + def get_docs_by_generation(self, encrypted=None): """ Get all documents in the received table from the sync db, ordered by generation. - :return: list of doc_id, rev, generation + :param encrypted: If not None, only return documents with encrypted + field equal to given parameter. + :type encrypted: bool + + :return: list of doc_id, rev, generation, gen, trans_id + :rtype: list """ + sql = "SELECT doc_id, rev, content, gen, trans_id, encrypted FROM %s" \ + % self.TABLE_NAME + if encrypted is not None: + sql += " WHERE encrypted = %d" % int(encrypted) + sql += " ORDER BY gen" c = self._sync_db.cursor() - sql = "SELECT doc_id, rev, gen FROM %s ORDER BY gen" % ( - self.TABLE_NAME,) c.execute(sql) - return c.fetchall() + # TODO: due to unknown reasons, the fetchall() method may return empty + # values, so we filter them out here. We have to perform some tests to + # understand why and when this happens. + docs = filter(lambda entry: len(entry) > 0, c.fetchall()) + return docs + + def get_insertable_docs_by_gen(self): + """ + Return a list of documents ready to be inserted. + """ + docs = self.get_docs_by_generation(encrypted=False) + insertable = [] + if docs: + last_gen = self._last_known_generation + for doc_id, rev, content, gen, trans_id, _ in docs: + if gen != (last_gen + 1): + break + insertable.append((doc_id, rev, content, gen, trans_id)) + last_gen = gen + return insertable - def count_received_encrypted_docs(self): + def count_docs_in_sync_db(self): """ - Count how many documents we have in the table for received and - encrypted docs. + Count how many documents we have in the table for received docs. :return: The count of documents. :rtype: int @@ -916,11 +926,13 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): Get all the encrypted documents from the sync database and dispatch a decrypt worker to decrypt each one of them. """ - docs_by_generation = self.get_docs_by_generation() + docs_by_generation = self.get_docs_by_generation(encrypted=True) logger.debug("Sync decrypter pool: There are %d documents to " \ "decrypt." % len(docs_by_generation)) - for doc_id, rev, gen in filter(None, docs_by_generation): - self.decrypt_doc(doc_id, rev, self.source_replica_uid) + for doc_id, rev, content, gen, trans_id, _ \ + in filter(None, docs_by_generation): + self.decrypt_doc( + doc_id, rev, content, gen, trans_id, self.source_replica_uid) def process_decrypted(self): """ @@ -934,15 +946,9 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): # getting data from the syncing stream, to avoid InvalidGeneration # problems. with self.write_encrypted_lock: - already_decrypted = self.decrypted_docs - docs = self.get_docs_by_generation() - docs = filter(lambda entry: len(entry) > 0, docs) - expected = [gen for doc_id, rev, gen in docs] - docs_to_insert = get_insertable_docs_by_gen( - expected, already_decrypted) - for doc_fields in docs_to_insert: + for doc_fields in self.get_insertable_docs_by_gen(): self.insert_decrypted_local_doc(*doc_fields) - remaining = self.count_received_encrypted_docs() + remaining = self.count_docs_in_sync_db() return remaining == 0 def insert_decrypted_local_doc(self, doc_id, doc_rev, content, @@ -974,14 +980,14 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): if content == 'null': content = None doc = SoledadDocument(doc_id, doc_rev, content) - insert_fun(doc, int(gen), trans_id) + gen = int(gen) + insert_fun(doc, gen, trans_id) + self._last_known_generation = gen except Exception as exc: logger.error("Sync decrypter pool: error while inserting " "decrypted doc into local db.") logger.exception(exc) else: - # If no errors found, remove it from the local temporary dict - # and from the received database. - self.decrypted_docs.pop(gen) - self.delete_encrypted_received_doc(doc_id, doc_rev) + # If no errors found, remove it from the received database. + self.delete_received_doc(doc_id, doc_rev) diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 70e4d3a2..089a48a0 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -804,16 +804,20 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): self._sync_db = sync_db self._sync_db_write_lock = sync_db_write_lock - def _setup_sync_decr_pool(self): + def _setup_sync_decr_pool(self, last_known_generation): """ Set up the SyncDecrypterPool for deferred decryption. + + :param last_known_generation: Target's last known generation. + :type last_known_generation: int """ if self._sync_decr_pool is None: # initialize syncing queue decryption pool self._sync_decr_pool = SyncDecrypterPool( self._crypto, self._sync_db, self._sync_db_write_lock, - insert_doc_cb=self._insert_doc_cb) + insert_doc_cb=self._insert_doc_cb, + last_known_generation=last_known_generation) self._sync_decr_pool.set_source_replica_uid( self.source_replica_uid) @@ -1127,7 +1131,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): if defer_decryption: self._sync_exchange_lock.acquire() - self._setup_sync_decr_pool() + self._setup_sync_decr_pool(last_known_generation) self._setup_sync_watcher() self._defer_decryption = True @@ -1402,7 +1406,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): :rtype: bool """ if self._sync_decr_pool is not None: - return self._sync_decr_pool.count_received_encrypted_docs() == 0 + return self._sync_decr_pool.count_docs_in_sync_db() == 0 else: return True -- cgit v1.2.3 From 5e4dae3427f40879156ddfaaaa8f878ab2504ee3 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 23 Jul 2014 10:29:09 -0300 Subject: On sync, fetch all docs before decrypting. --- client/src/leap/soledad/client/crypto.py | 30 +++++++++++++++++------------- client/src/leap/soledad/client/target.py | 32 +++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 4a73a910..5ae5937f 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -690,7 +690,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :type last_known_generation: int """ self._insert_doc_cb = kwargs.pop("insert_doc_cb") - self._last_known_generation = kwargs.pop("last_known_generation") SyncEncryptDecryptPool.__init__(self, *args, **kwargs) self.source_replica_uid = None @@ -858,8 +857,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :type result: tuple(str, str, str) """ doc_id, rev, content, gen, trans_id = result - logger.debug("Sync decrypter pool: decrypted doc %s: %s %s" - % (doc_id, rev, gen)) + logger.debug("Sync decrypter pool: decrypted doc %s: %s %s %s" + % (doc_id, rev, gen, trans_id)) self.insert_received_doc(doc_id, rev, content, gen, trans_id) def get_docs_by_generation(self, encrypted=None): @@ -878,7 +877,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): % self.TABLE_NAME if encrypted is not None: sql += " WHERE encrypted = %d" % int(encrypted) - sql += " ORDER BY gen" + sql += " ORDER BY gen ASC" c = self._sync_db.cursor() c.execute(sql) # TODO: due to unknown reasons, the fetchall() method may return empty @@ -891,21 +890,25 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ Return a list of documents ready to be inserted. """ - docs = self.get_docs_by_generation(encrypted=False) + all_docs = self.get_docs_by_generation() + decrypted_docs = self.get_docs_by_generation(encrypted=False) insertable = [] - if docs: - last_gen = self._last_known_generation - for doc_id, rev, content, gen, trans_id, _ in docs: - if gen != (last_gen + 1): - break + for doc_id, rev, content, gen, trans_id, encrypted in all_docs: + next_decrypted = decrypted_docs.pop(0) + if doc_id == next_decrypted[0]: insertable.append((doc_id, rev, content, gen, trans_id)) - last_gen = gen + else: + break return insertable - def count_docs_in_sync_db(self): + def count_docs_in_sync_db(self, encrypted=None): """ Count how many documents we have in the table for received docs. + :param encrypted: If not None, return count of documents with + encrypted field equal to given parameter. + :type encrypted: bool + :return: The count of documents. :rtype: int """ @@ -914,6 +917,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): return c = self._sync_db.cursor() sql = "SELECT COUNT(*) FROM %s" % (self.TABLE_NAME,) + if encrypted is not None: + sql += " WHERE encrypted = %d" % int(encrypted) c.execute(sql) res = c.fetchone() if res is not None: @@ -982,7 +987,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): doc = SoledadDocument(doc_id, doc_rev, content) gen = int(gen) insert_fun(doc, gen, trans_id) - self._last_known_generation = gen except Exception as exc: logger.error("Sync decrypter pool: error while inserting " "decrypted doc into local db.") diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 089a48a0..032134ec 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -816,8 +816,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): self._sync_decr_pool = SyncDecrypterPool( self._crypto, self._sync_db, self._sync_db_write_lock, - insert_doc_cb=self._insert_doc_cb, - last_known_generation=last_known_generation) + insert_doc_cb=self._insert_doc_cb) self._sync_decr_pool.set_source_replica_uid( self.source_replica_uid) @@ -1251,15 +1250,26 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): sent += 1 # make sure all threads finished and we have up-to-date info + last_successful_thread = None while threads: # check if there are failures t, doc = threads.pop(0) t.join() if t.success: synced.append((doc.doc_id, doc.rev)) + last_successful_thread = t - if defer_decryption: - self._sync_watcher.start() + # delete documents from the sync database + if defer_encryption: + self.delete_encrypted_docs_from_db(synced) + + # get target gen and trans_id after docs + gen_after_send = None + trans_id_after_send = None + if last_successful_thread is not None: + response_dict = json.loads(last_successful_thread.response[0])[0] + gen_after_send = response_dict['new_generation'] + trans_id_after_send = response_dict['new_transaction_id'] # get docs from target if self.stopped is False: @@ -1268,20 +1278,24 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): last_known_generation, last_known_trans_id, headers, return_doc_cb, ensure_callback, sync_id, syncer_pool, defer_decryption=defer_decryption) - syncer_pool.cleanup() - # delete documents from the sync database - if defer_encryption: - self.delete_encrypted_docs_from_db(synced) + syncer_pool.cleanup() - # wait for deferred decryption to finish + # decrypt docs in case of deferred decryption if defer_decryption: + self._sync_watcher.start() while self.clear_to_sync() is False: sleep(self.DECRYPT_TASK_PERIOD) self._teardown_sync_watcher() self._teardown_sync_decr_pool() self._sync_exchange_lock.release() + # update gen and trans id info in case we just sent and did not + # receive docs. + if gen_after_send is not None and gen_after_send > cur_target_gen: + cur_target_gen = gen_after_send + cur_target_trans_id = trans_id_after_send + self.stop() return cur_target_gen, cur_target_trans_id -- cgit v1.2.3 From 8afb79c4d2171b03270143639296cbb6d9d0fdfa Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 23 Jul 2014 10:29:39 -0300 Subject: Allow deferred decryption without deferred encryption. --- client/src/leap/soledad/client/sqlcipher.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 2df9606e..5a30b125 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -243,19 +243,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self._ensure_schema() self._crypto = crypto + # define sync-db attrs self._sync_db = None self._sync_db_write_lock = None self._sync_enc_pool = None + self._init_sync_db(sqlcipher_file) if self.defer_encryption: - if sqlcipher_file != ":memory:": - self._sync_db_path = "%s-sync" % sqlcipher_file - else: - self._sync_db_path = ":memory:" - # initialize sync db - self._init_sync_db() - # initialize syncing queue encryption pool self._sync_enc_pool = SyncEncrypterPool( self._crypto, self._sync_db, self._sync_db_write_lock) @@ -449,7 +444,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # the following context manager blocks until the syncing lock can be # acquired. with self.syncer(url, creds=creds) as syncer: - # XXX could mark the critical section here... try: res = syncer.sync(autocreate=autocreate, @@ -542,14 +536,21 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') - def _init_sync_db(self): + def _init_sync_db(self, sqlcipher_file): """ Initialize the Symmetrically-Encrypted document to be synced database, and the queue to communicate with subprocess workers. + + :param sqlcipher_file: The path for the SQLCipher file. + :type sqlcipher_file: str """ - self._sync_db = sqlite3.connect(self._sync_db_path, + sync_db_path = None + if sqlcipher_file != ":memory:": + sync_db_path = "%s-sync" % sqlcipher_file + else: + sync_db_path = ":memory:" + self._sync_db = sqlite3.connect(sync_db_path, check_same_thread=False) - self._sync_db_write_lock = threading.Lock() self._create_sync_db_tables() self.sync_queue = multiprocessing.Queue() -- cgit v1.2.3 From 609669077b2f7223c31feed3679c8fcd74ab9ba7 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 23 Jul 2014 10:49:44 -0300 Subject: Avoid deadlocks when cancelling failed sync threads. --- client/src/leap/soledad/client/target.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 032134ec..5fe55216 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -376,6 +376,12 @@ class DocumentSyncerPool(object): t.request_lock.release() t.callback_lock.acquire(False) # just in case t.callback_lock.release() + # release any blocking semaphores + for i in xrange(DocumentSyncerPool.POOL_SIZE): + try: + self._semaphore_pool.release() + except ValueError: + break logger.warning("Soledad sync: cancelled sync threads.") def cleanup(self): -- cgit v1.2.3 From 622708945d51a1e22dde95424a6214e8e67be180 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 23 Jul 2014 16:26:24 -0300 Subject: Make sync database multiprocessing-safe. --- client/src/leap/soledad/client/crypto.py | 46 +++++------- client/src/leap/soledad/client/mp_safe_db.py | 101 +++++++++++++++++++++++++++ client/src/leap/soledad/client/sqlcipher.py | 22 ++++-- client/src/leap/soledad/client/target.py | 17 ++--- 4 files changed, 143 insertions(+), 43 deletions(-) create mode 100644 client/src/leap/soledad/client/mp_safe_db.py diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 5ae5937f..eb5a4f64 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -224,7 +224,7 @@ class SoledadCrypto(object): The password is derived using HMAC having sha256 as underlying hash function. The key used for HMAC are the first - C{soledad.REMOTE_STORAGE_SECRET_KENGTH} bytes of Soledad's storage + C{soledad.REMOTE_STORAGE_SECRET_LENGTH} bytes of Soledad's storage secret stripped from the first MAC_KEY_LENGTH characters. The HMAC message is C{doc_id}. @@ -623,9 +623,8 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): con = self._sync_db with self._sync_db_write_lock: - with con: - con.execute(sql_del, (doc_id, )) - con.execute(sql_ins, (doc_id, doc_rev, content)) + con.execute(sql_del, (doc_id, )) + con.execute(sql_ins, (doc_id, doc_rev, content)) def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret): @@ -726,11 +725,10 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): con = self._sync_db with self._sync_db_write_lock: - with con: - con.execute(sql_del, (doc_id, )) - con.execute( - sql_ins, - (doc_id, doc_rev, docstr, gen, trans_id, 1)) + con.execute(sql_del, (doc_id, )) + con.execute( + sql_ins, + (doc_id, doc_rev, docstr, gen, trans_id, 1)) def insert_received_doc(self, doc_id, doc_rev, content, gen, trans_id): """ @@ -757,11 +755,10 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): self.TABLE_NAME,) con = self._sync_db with self._sync_db_write_lock: - with con: - con.execute(sql_del, (doc_id,)) - con.execute( - sql_ins, - (doc_id, doc_rev, content, gen, trans_id, 0)) + con.execute(sql_del, (doc_id,)) + con.execute( + sql_ins, + (doc_id, doc_rev, content, gen, trans_id, 0)) def delete_received_doc(self, doc_id, doc_rev): """ @@ -776,8 +773,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): self.TABLE_NAME,) con = self._sync_db with self._sync_db_write_lock: - with con: - con.execute(sql_del, (doc_id, doc_rev)) + con.execute(sql_del, (doc_id, doc_rev)) def decrypt_doc(self, doc_id, rev, content, gen, trans_id, source_replica_uid, workers=True): @@ -878,12 +874,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): if encrypted is not None: sql += " WHERE encrypted = %d" % int(encrypted) sql += " ORDER BY gen ASC" - c = self._sync_db.cursor() - c.execute(sql) - # TODO: due to unknown reasons, the fetchall() method may return empty - # values, so we filter them out here. We have to perform some tests to - # understand why and when this happens. - docs = filter(lambda entry: len(entry) > 0, c.fetchall()) + docs = self._sync_db.select(sql) return docs def get_insertable_docs_by_gen(self): @@ -894,7 +885,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): decrypted_docs = self.get_docs_by_generation(encrypted=False) insertable = [] for doc_id, rev, content, gen, trans_id, encrypted in all_docs: - next_decrypted = decrypted_docs.pop(0) + next_decrypted = decrypted_docs.next() if doc_id == next_decrypted[0]: insertable.append((doc_id, rev, content, gen, trans_id)) else: @@ -915,14 +906,13 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): if self._sync_db is None: logger.warning("cannot return count with null sync_db") return - c = self._sync_db.cursor() sql = "SELECT COUNT(*) FROM %s" % (self.TABLE_NAME,) if encrypted is not None: sql += " WHERE encrypted = %d" % int(encrypted) - c.execute(sql) - res = c.fetchone() + res = self._sync_db.select(sql) if res is not None: - return res[0] + val = res.next() + return val[0] else: return 0 @@ -932,8 +922,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): decrypt worker to decrypt each one of them. """ docs_by_generation = self.get_docs_by_generation(encrypted=True) - logger.debug("Sync decrypter pool: There are %d documents to " \ - "decrypt." % len(docs_by_generation)) for doc_id, rev, content, gen, trans_id, _ \ in filter(None, docs_by_generation): self.decrypt_doc( diff --git a/client/src/leap/soledad/client/mp_safe_db.py b/client/src/leap/soledad/client/mp_safe_db.py new file mode 100644 index 00000000..a9ab5649 --- /dev/null +++ b/client/src/leap/soledad/client/mp_safe_db.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# crypto.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Multiprocessing-safe SQLite database. +""" + + +from threading import Thread +from Queue import Queue +from sqlite3 import connect as sqlite3_connect + + +# Thanks to http://code.activestate.com/recipes/526618/ + +class MPSafeSQLiteDB(Thread): + """ + A multiprocessing-safe SQLite database accessor. + """ + + CLOSE = "--close--" + NO_MORE = "--no more--" + + def __init__(self, db_path): + """ + Initialize the process + """ + Thread.__init__(self) + self._db_path = db_path + self._requests = Queue() + self.start() + + def run(self): + """ + Run the multiprocessing-safe database accessor. + """ + conn = sqlite3_connect(self._db_path) + while True: + req, arg, res = self._requests.get() + if req == self.CLOSE: + break + with conn: + cursor = conn.cursor() + cursor.execute(req, arg) + if res: + for rec in cursor.fetchall(): + res.put(rec) + res.put(self.NO_MORE) + conn.close() + + def execute(self, req, arg=None, res=None): + """ + Execute a request on the database. + + :param req: The request to be executed. + :type req: str + :param arg: The arguments for the request. + :type arg: tuple + :param res: A queue to write request results. + :type res: multiprocessing.Queue + """ + self._requests.put((req, arg or tuple(), res)) + + def select(self, req, arg=None): + """ + Run a select query on the database and yield results. + + :param req: The request to be executed. + :type req: str + :param arg: The arguments for the request. + :type arg: tuple + """ + res = Queue() + self.execute(req, arg, res) + while True: + rec=res.get() + if rec == self.NO_MORE: + break + yield rec + + def close(self): + """ + Close the database connection. + """ + self.execute(self.CLOSE) + self.join() diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 5a30b125..85b0391b 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -44,7 +44,6 @@ handled by Soledad should be created by SQLCipher >= 2.0. import logging import multiprocessing import os -import sqlite3 import string import threading import time @@ -63,6 +62,7 @@ from leap.soledad.client.crypto import SyncEncrypterPool, SyncDecrypterPool from leap.soledad.client.target import SoledadSyncTarget from leap.soledad.client.target import PendingReceivedDocsSyncError from leap.soledad.client.sync import SoledadSynchronizer +from leap.soledad.client.mp_safe_db import MPSafeSQLiteDB from leap.soledad.common.document import SoledadDocument @@ -549,8 +549,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): sync_db_path = "%s-sync" % sqlcipher_file else: sync_db_path = ":memory:" - self._sync_db = sqlite3.connect(sync_db_path, - check_same_thread=False) + self._sync_db = MPSafeSQLiteDB(sync_db_path) self._sync_db_write_lock = threading.Lock() self._create_sync_db_tables() self.sync_queue = multiprocessing.Queue() @@ -567,9 +566,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): decr.TABLE_NAME, decr.FIELD_NAMES)) with self._sync_db_write_lock: - with self._sync_db: - self._sync_db.execute(sql_encr) - self._sync_db.execute(sql_decr) + self._sync_db.execute(sql_encr) + self._sync_db.execute(sql_decr) # # Symmetric encryption of syncing docs @@ -1076,16 +1074,28 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): Close db_handle and close syncer. """ logger.debug("Sqlcipher backend: closing") + # stop the sync watcher for deferred encryption if self._sync_watcher is not None: self._sync_watcher.stop() self._sync_watcher.shutdown() + # close all open syncers for url in self._syncers: _, syncer = self._syncers[url] syncer.close() + # stop the encryption pool if self._sync_enc_pool is not None: self._sync_enc_pool.close() + # close the actual database if self._db_handle is not None: self._db_handle.close() + # close the sync database + if self._sync_db is not None: + self._sync_db.close() + # close the sync queue + if self.sync_queue is not None: + self.sync_queue.close() + del self.sync_queue + self.sync_queue = None @property def replica_uid(self): diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 5fe55216..01e1231a 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -1346,13 +1346,16 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): :type doc_rev: str """ encr = SyncEncrypterPool - c = self._sync_db.cursor() sql = ("SELECT content FROM %s WHERE doc_id=? and rev=?" % ( encr.TABLE_NAME,)) - c.execute(sql, (doc_id, doc_rev)) - res = c.fetchall() - if len(res) != 0: - return res[0][0] + res = self._sync_db.select(sql, (doc_id, doc_rev)) + try: + val = res.next() + return val[0] + except StopIteration: + # no doc found + return None + def delete_encrypted_docs_from_db(self, docs_ids): """ @@ -1365,12 +1368,10 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): """ if docs_ids: encr = SyncEncrypterPool - c = self._sync_db.cursor() for doc_id, doc_rev in docs_ids: sql = ("DELETE FROM %s WHERE doc_id=? and rev=?" % ( encr.TABLE_NAME,)) - c.execute(sql, (doc_id, doc_rev)) - self._sync_db.commit() + self._sync_db.execute(sql, (doc_id, doc_rev)) def _save_encrypted_received_doc(self, doc, gen, trans_id, idx, total): """ -- cgit v1.2.3 From 074848f78bdac78328eb4de7fe72d85830da561d Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 1 Aug 2014 12:47:43 -0300 Subject: Refactor secrets out of main soledad client class. --- client/src/leap/soledad/client/__init__.py | 659 +++------------------------- client/src/leap/soledad/client/crypto.py | 21 +- client/src/leap/soledad/client/secrets.py | 681 +++++++++++++++++++++++++++++ 3 files changed, 757 insertions(+), 604 deletions(-) create mode 100644 client/src/leap/soledad/client/secrets.py diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 586e3389..0fd6672a 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -31,9 +31,7 @@ import os import socket import ssl import urlparse -import hmac -from hashlib import sha256 try: import cchardet as chardet @@ -43,41 +41,20 @@ except ImportError: from u1db.remote import http_client from u1db.remote.ssl_match_hostname import match_hostname -import scrypt -import simplejson as json - from leap.common.config import get_path_prefix from leap.soledad.common import ( SHARED_DB_NAME, soledad_assert, soledad_assert_type ) -from leap.soledad.common.errors import ( - InvalidTokenError, - NotLockedError, - AlreadyLockedError, - LockTimedOutError, -) -from leap.soledad.common.crypto import ( - MacMethods, - UnknownMacMethod, - WrongMac, - MAC_KEY, - MAC_METHOD_KEY, -) from leap.soledad.client.events import ( - SOLEDAD_CREATING_KEYS, - SOLEDAD_DONE_CREATING_KEYS, - SOLEDAD_DOWNLOADING_KEYS, - SOLEDAD_DONE_DOWNLOADING_KEYS, - SOLEDAD_UPLOADING_KEYS, - SOLEDAD_DONE_UPLOADING_KEYS, SOLEDAD_NEW_DATA_TO_SYNC, SOLEDAD_DONE_DATA_SYNC, signal, ) from leap.soledad.common.document import SoledadDocument from leap.soledad.client.crypto import SoledadCrypto +from leap.soledad.client.secrets import SoledadSecrets from leap.soledad.client.shared_db import SoledadSharedDatabase from leap.soledad.client.sqlcipher import open as sqlcipher_open from leap.soledad.client.sqlcipher import SQLCipherDatabase @@ -102,27 +79,6 @@ Soledad client and server. # Soledad: local encrypted storage and remote encrypted sync. # -class NoStorageSecret(Exception): - """ - Raised when trying to use a storage secret but none is available. - """ - pass - - -class PassphraseTooShort(Exception): - """ - Raised when trying to change the passphrase but the provided passphrase is - too short. - """ - - -class BootstrapSequenceError(Exception): - """ - Raised when an attempt to generate a secret and store it in a recovery - documents on server failed. - """ - - class Soledad(object): """ Soledad provides encrypted data storage and sync. @@ -166,57 +122,6 @@ class Soledad(object): The name of the file where the storage secrets will be stored. """ - GENERATED_SECRET_LENGTH = 1024 - """ - The length of the generated secret used to derive keys for symmetric - encryption for local and remote storage. - """ - - LOCAL_STORAGE_SECRET_LENGTH = 512 - """ - The length of the secret used to derive a passphrase for the SQLCipher - database. - """ - - REMOTE_STORAGE_SECRET_LENGTH = \ - GENERATED_SECRET_LENGTH - LOCAL_STORAGE_SECRET_LENGTH - """ - The length of the secret used to derive an encryption key and a MAC auth - key for remote storage. - """ - - SALT_LENGTH = 64 - """ - The length of the salt used to derive the key for the storage secret - encryption. - """ - - MINIMUM_PASSPHRASE_LENGTH = 6 - """ - The minimum length for a passphrase. The passphrase length is only checked - when the user changes her passphrase, not when she instantiates Soledad. - """ - - IV_SEPARATOR = ":" - """ - A separator used for storing the encryption initial value prepended to the - ciphertext. - """ - - UUID_KEY = 'uuid' - STORAGE_SECRETS_KEY = 'storage_secrets' - SECRET_KEY = 'secret' - CIPHER_KEY = 'cipher' - LENGTH_KEY = 'length' - KDF_KEY = 'kdf' - KDF_SALT_KEY = 'kdf_salt' - KDF_LENGTH_KEY = 'kdf_length' - KDF_SCRYPT = 'scrypt' - CIPHER_AES256 = 'aes256' - """ - Keys used to access storage secrets in recovery documents. - """ - DEFAULT_PREFIX = os.path.join(get_path_prefix(), 'leap', 'soledad') """ Prefix for default values for path. @@ -266,41 +171,49 @@ class Soledad(object): storage on server sequence has failed for some reason. """ - # get config params + # store config params self._uuid = uuid - soledad_assert_type(passphrase, unicode) self._passphrase = passphrase - # init crypto variables - self._secrets = {} - self._secret_id = secret_id + self._secrets_path = secrets_path + self._local_db_path = local_db_path + self._server_url = server_url + # configure SSL certificate + global SOLEDAD_CERT + SOLEDAD_CERT = cert_file + self._set_token(auth_token) self._defer_encryption = defer_encryption - self._init_config(secrets_path, local_db_path, server_url) + self._init_config() + self._init_dirs() - self._set_token(auth_token) + # init crypto variables self._shared_db_instance = None - # configure SSL certificate - global SOLEDAD_CERT - SOLEDAD_CERT = cert_file + self._crypto = SoledadCrypto(self) + self._secrets = SoledadSecrets( + self._uuid, + self._passphrase, + self._secrets_path, + self._shared_db, + self._crypto, + secret_id=secret_id) + # initiate bootstrap sequence self._bootstrap() # might raise BootstrapSequenceError() - def _init_config(self, secrets_path, local_db_path, server_url): + def _init_config(self): """ Initialize configuration using default values for missing params. """ + soledad_assert_type(self._passphrase, unicode) # initialize secrets_path - self._secrets_path = secrets_path if self._secrets_path is None: self._secrets_path = os.path.join( self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME) # initialize local_db_path - self._local_db_path = local_db_path if self._local_db_path is None: self._local_db_path = os.path.join( self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME) # initialize server_url - self._server_url = server_url soledad_assert( self._server_url is not None, 'Missing URL for Soledad server.') @@ -309,129 +222,18 @@ class Soledad(object): # initialization/destruction methods # - def _get_or_gen_crypto_secrets(self): - """ - Retrieves or generates the crypto secrets. - - Might raise BootstrapSequenceError - """ - doc = self._get_secrets_from_shared_db() - - if doc: - logger.info( - 'Found cryptographic secrets in shared recovery ' - 'database.') - _, mac = self.import_recovery_document(doc.content) - if mac is False: - self.put_secrets_in_shared_db() - self._store_secrets() # save new secrets in local file - if self._secret_id is None: - self._set_secret_id(self._secrets.items()[0][0]) - else: - # STAGE 3 - there are no secrets in server also, so - # generate a secret and store it in remote db. - logger.info( - 'No cryptographic secrets found, creating new ' - ' secrets...') - self._set_secret_id(self._gen_secret()) - try: - self._put_secrets_in_shared_db() - except Exception as ex: - # storing generated secret in shared db failed for - # some reason, so we erase the generated secret and - # raise. - try: - os.unlink(self._secrets_path) - except OSError as e: - if e.errno != errno.ENOENT: # no such file or directory - logger.exception(e) - logger.exception(ex) - raise BootstrapSequenceError( - 'Could not store generated secret in the shared ' - 'database, bailing out...') - def _bootstrap(self): """ Bootstrap local Soledad instance. - Soledad Client bootstrap is the following sequence of stages: - - * stage 0 - local environment setup. - - directory initialization. - - crypto submodule initialization - * stage 1 - local secret loading: - - if secrets exist locally, load them. - * stage 2 - remote secret loading: - - else, if secrets exist in server, download them. - * stage 3 - secret generation: - - else, generate a new secret and store in server. - * stage 4 - database initialization. - - This method decides which bootstrap stages have already been performed - and performs the missing ones in order. - :raise BootstrapSequenceError: Raised when the secret generation and storage on server sequence has failed for some reason. """ - # STAGE 0 - local environment setup - self._init_dirs() - self._crypto = SoledadCrypto(self) - - secrets_problem = None - - # STAGE 1 - verify if secrets exist locally - if not self._has_secret(): # try to load from local storage. - - # STAGE 2 - there are no secrets in local storage, so try to fetch - # encrypted secrets from server. - logger.info( - 'Trying to fetch cryptographic secrets from shared recovery ' - 'database...') - - # --- start of atomic operation in shared db --- - - # obtain lock on shared db - token = timeout = None - try: - token, timeout = self._shared_db.lock() - except AlreadyLockedError: - raise BootstrapSequenceError('Database is already locked.') - except LockTimedOutError: - raise BootstrapSequenceError('Lock operation timed out.') - - try: - self._get_or_gen_crypto_secrets() - except Exception as e: - secrets_problem = e - - # release the lock on shared db - try: - self._shared_db.unlock(token) - self._shared_db.close() - except NotLockedError: - # for some reason the lock expired. Despite that, secret - # loading or generation/storage must have been executed - # successfully, so we pass. - pass - except InvalidTokenError: - # here, our lock has not only expired but also some other - # client application has obtained a new lock and is currently - # doing its thing in the shared database. Using the same - # reasoning as above, we assume everything went smooth and - # pass. - pass - except Exception as e: - logger.error("Unhandled exception when unlocking shared " - "database.") - logger.exception(e) - - # --- end of atomic operation in shared db --- - - # STAGE 4 - local database initialization - if secrets_problem is None: + try: + self._secrets.bootstrap() self._init_db() - else: - raise secrets_problem + except: + raise def _init_dirs(self): """ @@ -460,27 +262,8 @@ class Soledad(object): Currently, Soledad uses the default SQLCipher cipher, i.e. 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key and uses the 'raw PRAGMA key' format to handle the key to SQLCipher. - - The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage - secret are used for remote storage encryption. We use the next - C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage. - From these bytes, the first C{self.SALT_LENGTH} are used as the salt - and the rest as the password for the scrypt hashing. - """ - # salt indexes - salt_start = self.REMOTE_STORAGE_SECRET_LENGTH - salt_end = salt_start + self.SALT_LENGTH - # password indexes - pwd_start = salt_end - pwd_end = salt_start + self.LOCAL_STORAGE_SECRET_LENGTH - # calculate the key for local encryption - secret = self._get_storage_secret() - key = scrypt.hash( - secret[pwd_start:pwd_end], # the password - secret[salt_start:salt_end], # the salt - buflen=32, # we need a key with 256 bits (32 bytes) - ) - + """ + key = self._secrets.get_local_storage_key() self._db = sqlcipher_open( self._local_db_path, binascii.b2a_hex(key), # sqlcipher only accepts the hex version @@ -501,186 +284,6 @@ class Soledad(object): self._db.stop_sync() self._db.close() - # - # Management of secret for symmetric encryption. - # - - def _get_storage_secret(self): - """ - Return the storage secret. - - Storage secret is encrypted before being stored. This method decrypts - and returns the stored secret. - - :return: The storage secret. - :rtype: str - """ - # calculate the encryption key - key = scrypt.hash( - self._passphrase_as_string(), - # the salt is stored base64 encoded - binascii.a2b_base64( - self._secrets[self._secret_id][self.KDF_SALT_KEY]), - buflen=32, # we need a key with 256 bits (32 bytes). - ) - # recover the initial value and ciphertext - iv, ciphertext = self._secrets[self._secret_id][self.SECRET_KEY].split( - self.IV_SEPARATOR, 1) - ciphertext = binascii.a2b_base64(ciphertext) - return self._crypto.decrypt_sym(ciphertext, key, iv=iv) - - def _set_secret_id(self, secret_id): - """ - Define the id of the storage secret to be used. - - This method will also replace the secret in the crypto object. - - :param secret_id: The id of the storage secret to be used. - :type secret_id: str - """ - self._secret_id = secret_id - - def _load_secrets(self): - """ - Load storage secrets from local file. - """ - # does the file exist in disk? - if not os.path.isfile(self._secrets_path): - raise IOError('File does not exist: %s' % self._secrets_path) - # read storage secrets from file - content = None - with open(self._secrets_path, 'r') as f: - content = json.loads(f.read()) - _, mac = self.import_recovery_document(content) - if mac is False: - self._store_secrets() - self._put_secrets_in_shared_db() - # choose first secret if no secret_id was given - if self._secret_id is None: - self._set_secret_id(self._secrets.items()[0][0]) - - def _has_secret(self): - """ - Return whether there is a storage secret available for use or not. - - :return: Whether there's a storage secret for symmetric encryption. - :rtype: bool - """ - if self._secret_id is None or self._secret_id not in self._secrets: - try: - self._load_secrets() # try to load from disk - except IOError, e: - logger.warning('IOError: %s' % str(e)) - try: - self._get_storage_secret() - return True - except Exception: - return False - - def _gen_secret(self): - """ - Generate a secret for symmetric encryption and store in a local - encrypted file. - - This method emits the following signals: - - * SOLEDAD_CREATING_KEYS - * SOLEDAD_DONE_CREATING_KEYS - - A secret has the following structure: - - { - '': { - 'kdf': 'scrypt', - 'kdf_salt': '' - 'kdf_length': - 'cipher': 'aes256', - 'length': , - 'secret': '', - } - } - - :return: The id of the generated secret. - :rtype: str - """ - signal(SOLEDAD_CREATING_KEYS, self._uuid) - # generate random secret - secret = os.urandom(self.GENERATED_SECRET_LENGTH) - secret_id = sha256(secret).hexdigest() - # generate random salt - salt = os.urandom(self.SALT_LENGTH) - # get a 256-bit key - key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) - iv, ciphertext = self._crypto.encrypt_sym(secret, key) - self._secrets[secret_id] = { - # leap.soledad.crypto submodule uses AES256 for symmetric - # encryption. - self.KDF_KEY: self.KDF_SCRYPT, - self.KDF_SALT_KEY: binascii.b2a_base64(salt), - self.KDF_LENGTH_KEY: len(key), - self.CIPHER_KEY: self.CIPHER_AES256, - self.LENGTH_KEY: len(secret), - self.SECRET_KEY: '%s%s%s' % ( - str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), - } - self._store_secrets() - signal(SOLEDAD_DONE_CREATING_KEYS, self._uuid) - return secret_id - - def _store_secrets(self): - """ - Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}. - """ - with open(self._secrets_path, 'w') as f: - f.write( - json.dumps( - self.export_recovery_document())) - - def change_passphrase(self, new_passphrase): - """ - Change the passphrase that encrypts the storage secret. - - :param new_passphrase: The new passphrase. - :type new_passphrase: unicode - - :raise NoStorageSecret: Raised if there's no storage secret available. - """ - # maybe we want to add more checks to guarantee passphrase is - # reasonable? - soledad_assert_type(new_passphrase, unicode) - if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH: - raise PassphraseTooShort( - 'Passphrase must be at least %d characters long!' % - self.MINIMUM_PASSPHRASE_LENGTH) - # ensure there's a secret for which the passphrase will be changed. - if not self._has_secret(): - raise NoStorageSecret() - secret = self._get_storage_secret() - # generate random salt - new_salt = os.urandom(self.SALT_LENGTH) - # get a 256-bit key - key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32) - iv, ciphertext = self._crypto.encrypt_sym(secret, key) - # XXX update all secrets in the dict - self._secrets[self._secret_id] = { - # leap.soledad.crypto submodule uses AES256 for symmetric - # encryption. - self.KDF_KEY: self.KDF_SCRYPT, # TODO: remove hard coded kdf - self.KDF_SALT_KEY: binascii.b2a_base64(new_salt), - self.KDF_LENGTH_KEY: len(key), - self.CIPHER_KEY: self.CIPHER_AES256, - self.LENGTH_KEY: len(secret), - self.SECRET_KEY: '%s%s%s' % ( - str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), - } - self._passphrase = new_passphrase - self._store_secrets() - self._put_secrets_in_shared_db() - - # - # General crypto utility methods. - # - @property def _shared_db(self): """ @@ -697,63 +300,6 @@ class Soledad(object): creds=self._creds) return self._shared_db_instance - def _shared_db_doc_id(self): - """ - Calculate the doc_id of the document in the shared db that stores key - material. - - :return: the hash - :rtype: str - """ - return sha256( - '%s%s' % - (self._passphrase_as_string(), self.uuid)).hexdigest() - - def _get_secrets_from_shared_db(self): - """ - Retrieve the document with encrypted key material from the shared - database. - - :return: a document with encrypted key material in its contents - :rtype: SoledadDocument - """ - signal(SOLEDAD_DOWNLOADING_KEYS, self._uuid) - db = self._shared_db - if not db: - logger.warning('No shared db found') - return - doc = db.get_doc(self._shared_db_doc_id()) - signal(SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid) - return doc - - def _put_secrets_in_shared_db(self): - """ - Assert local keys are the same as shared db's ones. - - Try to fetch keys from shared recovery database. If they already exist - in the remote db, assert that that data is the same as local data. - Otherwise, upload keys to shared recovery database. - """ - soledad_assert( - self._has_secret(), - 'Tried to send keys to server but they don\'t exist in local ' - 'storage.') - # try to get secrets doc from server, otherwise create it - doc = self._get_secrets_from_shared_db() - if doc is None: - doc = SoledadDocument( - doc_id=self._shared_db_doc_id()) - # fill doc with encrypted secrets - doc.content = self.export_recovery_document() - # upload secrets to server - signal(SOLEDAD_UPLOADING_KEYS, self._uuid) - db = self._shared_db - if not db: - logger.warning('No shared db found') - return - db.put_doc(doc) - signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid) - # # Document storage, retrieval and sync. # @@ -1152,104 +698,6 @@ class Soledad(object): token = property(_get_token, _set_token, doc='The authentication Token.') - # - # Recovery document export and import methods - # - - def export_recovery_document(self): - """ - Export the storage secrets. - - A recovery document has the following structure: - - { - 'storage_secrets': { - '': { - 'kdf': 'scrypt', - 'kdf_salt': '' - 'kdf_length': - 'cipher': 'aes256', - 'length': , - 'secret': '', - }, - }, - 'kdf': 'scrypt', - 'kdf_salt': '', - 'kdf_length: , - '_mac_method': 'hmac', - '_mac': '' - } - - Note that multiple storage secrets might be stored in one recovery - document. This method will also calculate a MAC of a string - representation of the secrets dictionary. - - :return: The recovery document. - :rtype: dict - """ - # create salt and key for calculating MAC - salt = os.urandom(self.SALT_LENGTH) - key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) - data = { - self.STORAGE_SECRETS_KEY: self._secrets, - self.KDF_KEY: self.KDF_SCRYPT, - self.KDF_SALT_KEY: binascii.b2a_base64(salt), - self.KDF_LENGTH_KEY: len(key), - MAC_METHOD_KEY: MacMethods.HMAC, - MAC_KEY: hmac.new( - key, - json.dumps(self._secrets), - sha256).hexdigest(), - } - return data - - def import_recovery_document(self, data): - """ - Import storage secrets for symmetric encryption and uuid (if present) - from a recovery document. - - Note that this method does not store the imported data on disk. For - that, use C{self._store_secrets()}. - - :param data: The recovery document. - :type data: dict - - :return: A tuple containing the number of imported secrets and whether - there was MAC informationa available for authenticating. - :rtype: (int, bool) - """ - soledad_assert(self.STORAGE_SECRETS_KEY in data) - # check mac of the recovery document - mac = None - if MAC_KEY in data: - soledad_assert(data[MAC_KEY] is not None) - soledad_assert(MAC_METHOD_KEY in data) - soledad_assert(self.KDF_KEY in data) - soledad_assert(self.KDF_SALT_KEY in data) - soledad_assert(self.KDF_LENGTH_KEY in data) - if data[MAC_METHOD_KEY] == MacMethods.HMAC: - key = scrypt.hash( - self._passphrase_as_string(), - binascii.a2b_base64(data[self.KDF_SALT_KEY]), - buflen=32) - mac = hmac.new( - key, - json.dumps(data[self.STORAGE_SECRETS_KEY]), - sha256).hexdigest() - else: - raise UnknownMacMethod('Unknown MAC method: %s.' % - data[MAC_METHOD_KEY]) - if mac != data[MAC_KEY]: - raise WrongMac('Could not authenticate recovery document\'s ' - 'contents.') - # include secrets in the secret pool. - secrets = 0 - for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items(): - if secret_id not in self._secrets: - secrets += 1 - self._secrets[secret_id] = secret_data - return secrets, mac - # # Setters/getters # @@ -1259,18 +707,26 @@ class Soledad(object): uuid = property(_get_uuid, doc='The user uuid.') - def _get_secret_id(self): - return self._secret_id + def get_secret_id(self): + return self._secrets.secret_id + + def set_secret_id(self, secret_id): + self._secrets.set_secret_id(secret_id) secret_id = property( - _get_secret_id, + get_secret_id, + set_secret_id, doc='The active secret id.') + def _set_secrets_path(self, secrets_path): + self._secrets.secrets_path = secrets_path + def _get_secrets_path(self): - return self._secrets_path + return self._secrets.secrets_path secrets_path = property( _get_secrets_path, + _set_secrets_path, doc='The path for the file containing the encrypted symmetric secret.') def _get_local_db_path(self): @@ -1287,20 +743,31 @@ class Soledad(object): _get_server_url, doc='The URL of the Soledad server.') - storage_secret = property( - _get_storage_secret, - doc='The secret used for symmetric encryption.') + @property + def storage_secret(self): + """ + Return the secret used for symmetric encryption. + """ + return self._secrets.storage_secret + + @property + def secrets(self): + return self._secrets - def _get_passphrase(self): - return self._passphrase + @property + def passphrase(self): + return self._secrets.passphrase - passphrase = property( - _get_passphrase, - doc='The passphrase for locking and unlocking encryption secrets for ' - 'local and remote storage.') + def change_passphrase(self, new_passphrase): + """ + Change the passphrase that encrypts the storage secret. - def _passphrase_as_string(self): - return self._passphrase.encode('utf-8') + :param new_passphrase: The new passphrase. + :type new_passphrase: unicode + + :raise NoStorageSecret: Raised if there's no storage secret available. + """ + self._secrets.change_passphrase(new_passphrase) # ---------------------------------------------------------------------------- diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index eb5a4f64..4a64b5a8 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -242,7 +242,7 @@ class SoledadCrypto(object): return hmac.new( self.secret[ MAC_KEY_LENGTH: - self._soledad.REMOTE_STORAGE_SECRET_LENGTH], + self._soledad.secrets.REMOTE_STORAGE_SECRET_LENGTH], doc_id, hashlib.sha256).digest() @@ -819,7 +819,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): try: content = json.loads(content) except TypeError: - logger.warning("Wrong type while decoding json: %s" % repr(docstr)) + logger.warning("Wrong type while decoding json: %s" + % repr(content)) return key = self._crypto.doc_passphrase(doc_id) @@ -884,11 +885,15 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): all_docs = self.get_docs_by_generation() decrypted_docs = self.get_docs_by_generation(encrypted=False) insertable = [] - for doc_id, rev, content, gen, trans_id, encrypted in all_docs: - next_decrypted = decrypted_docs.next() - if doc_id == next_decrypted[0]: - insertable.append((doc_id, rev, content, gen, trans_id)) - else: + for doc_id, rev, _, gen, trans_id, encrypted in all_docs: + try: + next_decrypted = decrypted_docs.next() + if doc_id == next_decrypted[0]: + content = next_decrypted[2] + insertable.append((doc_id, rev, content, gen, trans_id)) + else: + break + except StopIteration: break return insertable @@ -966,7 +971,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ # could pass source_replica in params for callback chain insert_fun = self._insert_doc_cb[self.source_replica_uid] - logger.debug("Sync decrypter pool: inserting doc in local db: " \ + logger.debug("Sync decrypter pool: inserting doc in local db: " "%s:%s %s" % (doc_id, doc_rev, gen)) try: # convert deleted documents to avoid error on document creation diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py new file mode 100644 index 00000000..3c6fc569 --- /dev/null +++ b/client/src/leap/soledad/client/secrets.py @@ -0,0 +1,681 @@ +# -*- coding: utf-8 -*- +# secrets.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Soledad secrets handling. +""" + + +import os +import scrypt +import hmac +import logging +import binascii +import errno + + +from hashlib import sha256 +import simplejson as json + + +from leap.soledad.common import ( + soledad_assert, + soledad_assert_type +) +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import ( + MacMethods, + UnknownMacMethod, + WrongMac, + MAC_KEY, + MAC_METHOD_KEY, +) +from leap.soledad.common.errors import ( + InvalidTokenError, + NotLockedError, + AlreadyLockedError, + LockTimedOutError, +) +from leap.soledad.client.events import ( + SOLEDAD_CREATING_KEYS, + SOLEDAD_DONE_CREATING_KEYS, + SOLEDAD_DOWNLOADING_KEYS, + SOLEDAD_DONE_DOWNLOADING_KEYS, + SOLEDAD_UPLOADING_KEYS, + SOLEDAD_DONE_UPLOADING_KEYS, + signal, +) + + +logger = logging.getLogger(name=__name__) + + +# +# Exceptions +# + +class NoStorageSecret(Exception): + """ + Raised when trying to use a storage secret but none is available. + """ + pass + + +class PassphraseTooShort(Exception): + """ + Raised when trying to change the passphrase but the provided passphrase is + too short. + """ + + +class BootstrapSequenceError(Exception): + """ + Raised when an attempt to generate a secret and store it in a recovery + document on server failed. + """ + + +# +# Secrets handler +# + +class SoledadSecrets(object): + """ + Soledad secrets handler. + + The first C{self.REMOTE_STORAGE_SECRET_LENGTH} bytes of the storage + secret are used for remote storage encryption. We use the next + C{self.LOCAL_STORAGE_SECRET} bytes to derive a key for local storage. + From these bytes, the first C{self.SALT_LENGTH} bytes are used as the + salt and the rest as the password for the scrypt hashing. + """ + + LOCAL_STORAGE_SECRET_LENGTH = 512 + """ + The length of the secret used to derive a passphrase for the SQLCipher + database. + """ + + REMOTE_STORAGE_SECRET_LENGTH = 512 + """ + The length of the secret used to derive an encryption key and a MAC auth + key for remote storage. + """ + + SALT_LENGTH = 64 + """ + The length of the salt used to derive the key for the storage secret + encryption. + """ + + MINIMUM_PASSPHRASE_LENGTH = 6 + """ + The minimum length for a passphrase. The passphrase length is only checked + when the user changes her passphrase, not when she instantiates Soledad. + """ + + IV_SEPARATOR = ":" + """ + A separator used for storing the encryption initial value prepended to the + ciphertext. + """ + + UUID_KEY = 'uuid' + STORAGE_SECRETS_KEY = 'storage_secrets' + SECRET_KEY = 'secret' + CIPHER_KEY = 'cipher' + LENGTH_KEY = 'length' + KDF_KEY = 'kdf' + KDF_SALT_KEY = 'kdf_salt' + KDF_LENGTH_KEY = 'kdf_length' + KDF_SCRYPT = 'scrypt' + CIPHER_AES256 = 'aes256' + """ + Keys used to access storage secrets in recovery documents. + """ + + def __init__(self, uuid, passphrase, secrets_path, shared_db, crypto, + secret_id=None): + """ + Initialize the secrets manager. + + :param uuid: User's unique id. + :type uuid: str + :param passphrase: The passphrase for locking and unlocking encryption + secrets for local and remote storage. + :type passphrase: unicode + :param secrets_path: Path for storing encrypted key used for + symmetric encryption. + :type secrets_path: str + :param shared_db: The shared database that stores user secrets. + :type shared_db: leap.soledad.client.shared_db.SoledadSharedDatabase + :param crypto: A soledad crypto object. + :type crypto: SoledadCrypto + :param secret_id: The id of the storage secret to be used. + :type secret_id: str + """ + self._uuid = uuid + self._passphrase = passphrase + self._secrets_path = secrets_path + self._shared_db = shared_db + self._crypto = crypto + self._secret_id = secret_id + self._secrets = {} + + def bootstrap(self): + """ + Bootstrap secrets. + + Soledad secrets bootstrap is the following sequence of stages: + + * stage 1 - local secret loading: + - if secrets exist locally, load them. + * stage 2 - remote secret loading: + - else, if secrets exist in server, download them. + * stage 3 - secret generation: + - else, generate a new secret and store in server. + + This method decides which bootstrap stages have already been performed + and performs the missing ones in order. + + :raise BootstrapSequenceError: Raised when the secret generation and + storage on server sequence has failed for some reason. + """ + # STAGE 1 - verify if secrets exist locally + if not self._has_secret(): # try to load from local storage. + + # STAGE 2 - there are no secrets in local storage, so try to fetch + # encrypted secrets from server. + logger.info( + 'Trying to fetch cryptographic secrets from shared recovery ' + 'database...') + + # --- start of atomic operation in shared db --- + + # obtain lock on shared db + token = timeout = None + try: + token, timeout = self._shared_db.lock() + except AlreadyLockedError: + raise BootstrapSequenceError('Database is already locked.') + except LockTimedOutError: + raise BootstrapSequenceError('Lock operation timed out.') + + self._get_or_gen_crypto_secrets() + + # release the lock on shared db + try: + self._shared_db.unlock(token) + self._shared_db.close() + except NotLockedError: + # for some reason the lock expired. Despite that, secret + # loading or generation/storage must have been executed + # successfully, so we pass. + pass + except InvalidTokenError: + # here, our lock has not only expired but also some other + # client application has obtained a new lock and is currently + # doing its thing in the shared database. Using the same + # reasoning as above, we assume everything went smooth and + # pass. + pass + except Exception as e: + logger.error("Unhandled exception when unlocking shared " + "database.") + logger.exception(e) + + # --- end of atomic operation in shared db --- + + def _has_secret(self): + """ + Return whether there is a storage secret available for use or not. + + :return: Whether there's a storage secret for symmetric encryption. + :rtype: bool + """ + if self._secret_id is None or self._secret_id not in self._secrets: + try: + self._load_secrets() # try to load from disk + except IOError as e: + logger.warning('IOError: %s' % str(e)) + try: + self.storage_secret + return True + except Exception as e: + logger.warning("Couldn't load storage secret: %s" % str(e)) + return False + + def _load_secrets(self): + """ + Load storage secrets from local file. + """ + # does the file exist in disk? + if not os.path.isfile(self._secrets_path): + raise IOError('File does not exist: %s' % self._secrets_path) + # read storage secrets from file + content = None + with open(self._secrets_path, 'r') as f: + content = json.loads(f.read()) + _, mac = self._import_recovery_document(content) + if mac is False: + self._store_secrets() + self._put_secrets_in_shared_db() + # choose first secret if no secret_id was given + if self._secret_id is None: + self.set_secret_id(self._secrets.items()[0][0]) + + def _get_or_gen_crypto_secrets(self): + """ + Retrieves or generates the crypto secrets. + + :raises BootstrapSequenceError: Raised when unable to store secrets in + shared database. + """ + doc = self._get_secrets_from_shared_db() + + if doc: + logger.info( + 'Found cryptographic secrets in shared recovery ' + 'database.') + _, mac = self._import_recovery_document(doc.content) + if mac is False: + self.put_secrets_in_shared_db() + self._store_secrets() # save new secrets in local file + if self._secret_id is None: + self.set_secret_id(self._secrets.items()[0][0]) + else: + # STAGE 3 - there are no secrets in server also, so + # generate a secret and store it in remote db. + logger.info( + 'No cryptographic secrets found, creating new ' + ' secrets...') + self.set_secret_id(self._gen_secret()) + try: + self._put_secrets_in_shared_db() + except Exception as ex: + # storing generated secret in shared db failed for + # some reason, so we erase the generated secret and + # raise. + try: + os.unlink(self._secrets_path) + except OSError as e: + if e.errno != errno.ENOENT: # no such file or directory + logger.exception(e) + logger.exception(ex) + raise BootstrapSequenceError( + 'Could not store generated secret in the shared ' + 'database, bailing out...') + + # + # Shared DB related methods + # + + def _shared_db_doc_id(self): + """ + Calculate the doc_id of the document in the shared db that stores key + material. + + :return: the hash + :rtype: str + """ + return sha256( + '%s%s' % + (self._passphrase_as_string(), self._uuid)).hexdigest() + + def _export_recovery_document(self): + """ + Export the storage secrets. + + A recovery document has the following structure: + + { + 'storage_secrets': { + '': { + 'kdf': 'scrypt', + 'kdf_salt': '' + 'kdf_length': + 'cipher': 'aes256', + 'length': , + 'secret': '', + }, + }, + 'kdf': 'scrypt', + 'kdf_salt': '', + 'kdf_length: , + '_mac_method': 'hmac', + '_mac': '' + } + + Note that multiple storage secrets might be stored in one recovery + document. This method will also calculate a MAC of a string + representation of the secrets dictionary. + + :return: The recovery document. + :rtype: dict + """ + # create salt and key for calculating MAC + salt = os.urandom(self.SALT_LENGTH) + key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) + data = { + self.STORAGE_SECRETS_KEY: self._secrets, + self.KDF_KEY: self.KDF_SCRYPT, + self.KDF_SALT_KEY: binascii.b2a_base64(salt), + self.KDF_LENGTH_KEY: len(key), + MAC_METHOD_KEY: MacMethods.HMAC, + MAC_KEY: hmac.new( + key, + json.dumps(self._secrets), + sha256).hexdigest(), + } + return data + + def _import_recovery_document(self, data): + """ + Import storage secrets for symmetric encryption and uuid (if present) + from a recovery document. + + Note that this method does not store the imported data on disk. For + that, use C{self._store_secrets()}. + + :param data: The recovery document. + :type data: dict + + :return: A tuple containing the number of imported secrets and whether + there was MAC informationa available for authenticating. + :rtype: (int, bool) + """ + soledad_assert(self.STORAGE_SECRETS_KEY in data) + # check mac of the recovery document + mac = None + if MAC_KEY in data: + soledad_assert(data[MAC_KEY] is not None) + soledad_assert(MAC_METHOD_KEY in data) + soledad_assert(self.KDF_KEY in data) + soledad_assert(self.KDF_SALT_KEY in data) + soledad_assert(self.KDF_LENGTH_KEY in data) + if data[MAC_METHOD_KEY] == MacMethods.HMAC: + key = scrypt.hash( + self._passphrase_as_string(), + binascii.a2b_base64(data[self.KDF_SALT_KEY]), + buflen=32) + mac = hmac.new( + key, + json.dumps(data[self.STORAGE_SECRETS_KEY]), + sha256).hexdigest() + else: + raise UnknownMacMethod('Unknown MAC method: %s.' % + data[MAC_METHOD_KEY]) + if mac != data[MAC_KEY]: + raise WrongMac('Could not authenticate recovery document\'s ' + 'contents.') + # include secrets in the secret pool. + secrets = 0 + for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items(): + if secret_id not in self._secrets: + secrets += 1 + self._secrets[secret_id] = secret_data + return secrets, mac + + def _get_secrets_from_shared_db(self): + """ + Retrieve the document with encrypted key material from the shared + database. + + :return: a document with encrypted key material in its contents + :rtype: SoledadDocument + """ + signal(SOLEDAD_DOWNLOADING_KEYS, self._uuid) + db = self._shared_db + if not db: + logger.warning('No shared db found') + return + doc = db.get_doc(self._shared_db_doc_id()) + signal(SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid) + return doc + + def _put_secrets_in_shared_db(self): + """ + Assert local keys are the same as shared db's ones. + + Try to fetch keys from shared recovery database. If they already exist + in the remote db, assert that that data is the same as local data. + Otherwise, upload keys to shared recovery database. + """ + soledad_assert( + self._has_secret(), + 'Tried to send keys to server but they don\'t exist in local ' + 'storage.') + # try to get secrets doc from server, otherwise create it + doc = self._get_secrets_from_shared_db() + if doc is None: + doc = SoledadDocument( + doc_id=self._shared_db_doc_id()) + # fill doc with encrypted secrets + doc.content = self._export_recovery_document() + # upload secrets to server + signal(SOLEDAD_UPLOADING_KEYS, self._uuid) + db = self._shared_db + if not db: + logger.warning('No shared db found') + return + db.put_doc(doc) + signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid) + + # + # Management of secret for symmetric encryption. + # + + @property + def storage_secret(self): + """ + Return the storage secret. + + Storage secret is encrypted before being stored. This method decrypts + and returns the stored secret. + + :return: The storage secret. + :rtype: str + """ + # calculate the encryption key + key = scrypt.hash( + self._passphrase_as_string(), + # the salt is stored base64 encoded + binascii.a2b_base64( + self._secrets[self._secret_id][self.KDF_SALT_KEY]), + buflen=32, # we need a key with 256 bits (32 bytes). + ) + # recover the initial value and ciphertext + iv, ciphertext = self._secrets[self._secret_id][self.SECRET_KEY].split( + self.IV_SEPARATOR, 1) + ciphertext = binascii.a2b_base64(ciphertext) + return self._crypto.decrypt_sym(ciphertext, key, iv=iv) + + def set_secret_id(self, secret_id): + """ + Define the id of the storage secret to be used. + + This method will also replace the secret in the crypto object. + + :param secret_id: The id of the storage secret to be used. + :type secret_id: str + """ + self._secret_id = secret_id + + def _gen_secret(self): + """ + Generate a secret for symmetric encryption and store in a local + encrypted file. + + This method emits the following signals: + + * SOLEDAD_CREATING_KEYS + * SOLEDAD_DONE_CREATING_KEYS + + A secret has the following structure: + + { + '': { + 'kdf': 'scrypt', + 'kdf_salt': '' + 'kdf_length': + 'cipher': 'aes256', + 'length': , + 'secret': '', + } + } + + :return: The id of the generated secret. + :rtype: str + """ + signal(SOLEDAD_CREATING_KEYS, self._uuid) + # generate random secret + secret = os.urandom( + self.LOCAL_STORAGE_SECRET_LENGTH + + self.REMOTE_STORAGE_SECRET_LENGTH) + secret_id = sha256(secret).hexdigest() + # generate random salt + salt = os.urandom(self.SALT_LENGTH) + # get a 256-bit key + key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) + iv, ciphertext = self._crypto.encrypt_sym(secret, key) + self._secrets[secret_id] = { + # leap.soledad.crypto submodule uses AES256 for symmetric + # encryption. + self.KDF_KEY: self.KDF_SCRYPT, + self.KDF_SALT_KEY: binascii.b2a_base64(salt), + self.KDF_LENGTH_KEY: len(key), + self.CIPHER_KEY: self.CIPHER_AES256, + self.LENGTH_KEY: len(secret), + self.SECRET_KEY: '%s%s%s' % ( + str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), + } + self._store_secrets() + signal(SOLEDAD_DONE_CREATING_KEYS, self._uuid) + return secret_id + + def _store_secrets(self): + """ + Store secrets in C{Soledad.STORAGE_SECRETS_FILE_PATH}. + """ + with open(self._secrets_path, 'w') as f: + f.write( + json.dumps( + self._export_recovery_document())) + + def change_passphrase(self, new_passphrase): + """ + Change the passphrase that encrypts the storage secret. + + :param new_passphrase: The new passphrase. + :type new_passphrase: unicode + + :raise NoStorageSecret: Raised if there's no storage secret available. + """ + # TODO: maybe we want to add more checks to guarantee passphrase is + # reasonable? + soledad_assert_type(new_passphrase, unicode) + if len(new_passphrase) < self.MINIMUM_PASSPHRASE_LENGTH: + raise PassphraseTooShort( + 'Passphrase must be at least %d characters long!' % + self.MINIMUM_PASSPHRASE_LENGTH) + # ensure there's a secret for which the passphrase will be changed. + if not self._has_secret(): + raise NoStorageSecret() + secret = self.storage_secret + # generate random salt + new_salt = os.urandom(self.SALT_LENGTH) + # get a 256-bit key + key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32) + iv, ciphertext = self._crypto.encrypt_sym(secret, key) + # XXX update all secrets in the dict + self._secrets[self._secret_id] = { + # leap.soledad.crypto submodule uses AES256 for symmetric + # encryption. + self.KDF_KEY: self.KDF_SCRYPT, # TODO: remove hard coded kdf + self.KDF_SALT_KEY: binascii.b2a_base64(new_salt), + self.KDF_LENGTH_KEY: len(key), + self.CIPHER_KEY: self.CIPHER_AES256, + self.LENGTH_KEY: len(secret), + self.SECRET_KEY: '%s%s%s' % ( + str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), + } + self._passphrase = new_passphrase + self._store_secrets() + self._put_secrets_in_shared_db() + + # + # Setters and getters + # + + @property + def secret_id(self): + return self._secret_id + + def _get_secrets_path(self): + return self._secrets_path + + def _set_secrets_path(self, secrets_path): + self._secrets_path = secrets_path + + secrets_path = property( + _get_secrets_path, + _set_secrets_path, + doc='The path for the file containing the encrypted symmetric secret.') + + @property + def passphrase(self): + """ + Return the passphrase for locking and unlocking encryption secrets for + local and remote storage. + """ + return self._passphrase + + def _passphrase_as_string(self): + return self._passphrase.encode('utf-8') + + def get_syncdb_secret(self): + """ + Return the secret for sync db. + """ + # TODO: implement. + pass + + def get_remote_secret(self): + """ + Return the secret for remote storage. + """ + # TODO: implement + pass + + def get_local_storage_key(self): + """ + Return the local storage key derived from the local storage secret. + """ + # salt indexes + salt_start = self.REMOTE_STORAGE_SECRET_LENGTH + salt_end = salt_start + self.SALT_LENGTH + # password indexes + pwd_start = salt_end + pwd_end = salt_start + self.LOCAL_STORAGE_SECRET_LENGTH + # calculate the key for local encryption + secret = self.storage_secret + return scrypt.hash( + secret[pwd_start:pwd_end], # the password + secret[salt_start:salt_end], # the salt + buflen=32, # we need a key with 256 bits (32 bytes) + ) -- cgit v1.2.3 From 21a3f854c07c1d40d50da8c922e956d3247a08b2 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 1 Aug 2014 12:53:05 -0300 Subject: Fix tests after many changes in client. --- client/src/leap/soledad/client/sync.py | 2 - client/src/leap/soledad/client/target.py | 34 ++++---- .../src/leap/soledad/common/tests/test_crypto.py | 62 +++++++------- .../src/leap/soledad/common/tests/test_server.py | 25 +++--- .../src/leap/soledad/common/tests/test_soledad.py | 95 +++++++++++----------- .../leap/soledad/common/tests/test_sqlcipher.py | 67 +++++++++++---- .../soledad/common/tests/test_sync_deferred.py | 10 +-- .../leap/soledad/common/tests/test_sync_target.py | 30 ++----- .../src/leap/soledad/common/tests/test_target.py | 19 +++-- .../common/tests/u1db_tests/test_backends.py | 1 + 10 files changed, 180 insertions(+), 165 deletions(-) diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index 5d545a77..c158f2a7 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -29,8 +29,6 @@ Extend u1db Synchronizer with the ability to: """ -import json - import logging import traceback from threading import Lock diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 01e1231a..12175f48 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -28,12 +28,10 @@ import logging import re import urllib import threading -import urlparse from collections import defaultdict from time import sleep from uuid import uuid4 -from contextlib import contextmanager import simplejson as json from taskthread import TimerTask @@ -44,7 +42,6 @@ from u1db.remote.http_client import _encode_query_parameter, HTTPClientBase from zope.proxy import ProxyBase from zope.proxy import sameProxiedObjects, setProxiedObject -from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument from leap.soledad.client.auth import TokenBasedAuth from leap.soledad.client.crypto import is_symmetrically_encrypted @@ -87,7 +84,7 @@ class DocumentSyncerThread(threading.Thread): """ def __init__(self, doc_syncer, release_method, failed_method, - idx, total, last_request_lock=None, last_callback_lock=None): + idx, total, last_request_lock=None, last_callback_lock=None): """ Initialize a new syncer thread. @@ -246,7 +243,7 @@ class DocumentSyncerPool(object): """ def __init__(self, raw_url, raw_creds, query_string, headers, - ensure_callback, stop_method): + ensure_callback, stop_method): """ Initialize the document syncer pool. @@ -279,7 +276,7 @@ class DocumentSyncerPool(object): self._threads = [] def new_syncer_thread(self, idx, total, last_request_lock=None, - last_callback_lock=None): + last_callback_lock=None): """ Yield a new document syncer thread. @@ -619,7 +616,7 @@ class HTTPDocumentSyncer(HTTPClientBase, TokenBasedAuth): self._conn.endheaders() def _get_doc(self, received, sync_id, last_known_generation, - last_known_trans_id): + last_known_trans_id): """ Get a sync document from server by means of a POST request. @@ -658,7 +655,7 @@ class HTTPDocumentSyncer(HTTPClientBase, TokenBasedAuth): return self._response() def _put_doc(self, sync_id, last_known_generation, last_known_trans_id, - id, rev, content, gen, trans_id, number_of_docs, doc_idx): + id, rev, content, gen, trans_id, number_of_docs, doc_idx): """ Put a sync document on server by means of a POST request. @@ -765,7 +762,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): # def __init__(self, url, source_replica_uid=None, creds=None, crypto=None, - sync_db=None, sync_db_write_lock=None): + sync_db=None, sync_db_write_lock=None): """ Initialize the SoledadSyncTarget. @@ -925,7 +922,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): """ new_generation, new_transaction_id, number_of_changes, doc_id, \ rev, content, gen, trans_id = \ - self._parse_received_doc_response(response) + self._parse_received_doc_response(response) if doc_id is not None: # decrypt incoming document and insert into local database # ------------------------------------------------------------- @@ -1134,11 +1131,14 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): """ self._ensure_callback = ensure_callback - if defer_decryption: + if defer_decryption and self._sync_db is not None: self._sync_exchange_lock.acquire() self._setup_sync_decr_pool(last_known_generation) self._setup_sync_watcher() self._defer_decryption = True + else: + # fall back + defer_decryption = False self.start() @@ -1149,7 +1149,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): setProxiedObject(self._insert_doc_cb[source_replica_uid], return_doc_cb) - if not self.clear_to_sync(): + if defer_decryption is True and not self.clear_to_sync(): raise PendingReceivedDocsSyncError self._ensure_connection() @@ -1171,7 +1171,6 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): self._raw_url, self._raw_creds, url, headers, ensure_callback, self.stop) threads = [] - last_request_lock = None last_callback_lock = None sent = 0 total = len(docs_by_generations) @@ -1227,7 +1226,8 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): t.doc_syncer.set_request_method( 'put', sync_id, cur_target_gen, cur_target_trans_id, id=doc.doc_id, rev=doc.rev, content=doc_json, gen=gen, - trans_id=trans_id, number_of_docs=number_of_docs, doc_idx=sent + 1) + trans_id=trans_id, number_of_docs=number_of_docs, + doc_idx=sent + 1) # set the success calback def _success_callback(idx, total, response): @@ -1251,7 +1251,6 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): # save thread and append t.start() threads.append((t, doc)) - last_request_lock = t.request_lock last_callback_lock = t.callback_lock sent += 1 @@ -1275,7 +1274,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): if last_successful_thread is not None: response_dict = json.loads(last_successful_thread.response[0])[0] gen_after_send = response_dict['new_generation'] - trans_id_after_send = response_dict['new_transaction_id'] + trans_id_after_send = response_dict['new_transaction_id'] # get docs from target if self.stopped is False: @@ -1356,7 +1355,6 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): # no doc found return None - def delete_encrypted_docs_from_db(self, docs_ids): """ Delete several encrypted documents from the database of symmetrically @@ -1467,7 +1465,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): decrypter = self._sync_decr_pool decrypter.decrypt_received_docs() - done = decrypter.process_decrypted() + decrypter.process_decrypted() def _sign_request(self, method, url_query, params): """ diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index 1071af14..ccff5e46 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -57,23 +57,23 @@ class EncryptedSyncTestCase(BaseSoledadTest): class RecoveryDocumentTestCase(BaseSoledadTest): def test_export_recovery_document_raw(self): - rd = self._soledad.export_recovery_document() - secret_id = rd[self._soledad.STORAGE_SECRETS_KEY].items()[0][0] - secret = rd[self._soledad.STORAGE_SECRETS_KEY][secret_id] - self.assertEqual(secret_id, self._soledad._secret_id) - self.assertEqual(secret, self._soledad._secrets[secret_id]) - self.assertTrue(self._soledad.CIPHER_KEY in secret) - self.assertTrue(secret[self._soledad.CIPHER_KEY] == 'aes256') - self.assertTrue(self._soledad.LENGTH_KEY in secret) - self.assertTrue(self._soledad.SECRET_KEY in secret) + rd = self._soledad.secrets._export_recovery_document() + secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0] + secret = rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] + self.assertEqual(secret_id, self._soledad.secrets._secret_id) + self.assertEqual(secret, self._soledad.secrets._secrets[secret_id]) + self.assertTrue(self._soledad.secrets.CIPHER_KEY in secret) + self.assertTrue(secret[self._soledad.secrets.CIPHER_KEY] == 'aes256') + self.assertTrue(self._soledad.secrets.LENGTH_KEY in secret) + self.assertTrue(self._soledad.secrets.SECRET_KEY in secret) def test_import_recovery_document(self): - rd = self._soledad.export_recovery_document() + rd = self._soledad.secrets._export_recovery_document() s = self._soledad_instance() - s.import_recovery_document(rd) - s._set_secret_id(self._soledad._secret_id) - self.assertEqual(self._soledad._get_storage_secret(), - s._get_storage_secret(), + s.secrets._import_recovery_document(rd) + s.set_secret_id(self._soledad.secrets._secret_id) + self.assertEqual(self._soledad.storage_secret, + s.storage_secret, 'Failed settinng secret for symmetric encryption.') s.close() @@ -83,13 +83,13 @@ class SoledadSecretsTestCase(BaseSoledadTest): def test__gen_secret(self): # instantiate and save secret_id sol = self._soledad_instance(user='user@leap.se') - self.assertTrue(len(sol._secrets) == 1) + self.assertTrue(len(sol.secrets._secrets) == 1) secret_id_1 = sol.secret_id # assert id is hash of secret self.assertTrue( secret_id_1 == hashlib.sha256(sol.storage_secret).hexdigest()) # generate new secret - secret_id_2 = sol._gen_secret() + secret_id_2 = sol.secrets._gen_secret() self.assertTrue(secret_id_1 != secret_id_2) sol.close() # re-instantiate @@ -97,18 +97,20 @@ class SoledadSecretsTestCase(BaseSoledadTest): user='user@leap.se', secret_id=secret_id_1) # assert ids are valid - self.assertTrue(len(sol._secrets) == 2) - self.assertTrue(secret_id_1 in sol._secrets) - self.assertTrue(secret_id_2 in sol._secrets) + self.assertTrue(len(sol.secrets._secrets) == 2) + self.assertTrue(secret_id_1 in sol.secrets._secrets) + self.assertTrue(secret_id_2 in sol.secrets._secrets) # assert format of secret 1 self.assertTrue(sol.storage_secret is not None) self.assertIsInstance(sol.storage_secret, str) - self.assertTrue(len(sol.storage_secret) == sol.GENERATED_SECRET_LENGTH) + secret_length = sol.secrets.LOCAL_STORAGE_SECRET_LENGTH \ + + sol.secrets.REMOTE_STORAGE_SECRET_LENGTH + self.assertTrue(len(sol.storage_secret) == secret_length) # assert format of secret 2 - sol._set_secret_id(secret_id_2) + sol.set_secret_id(secret_id_2) self.assertTrue(sol.storage_secret is not None) self.assertIsInstance(sol.storage_secret, str) - self.assertTrue(len(sol.storage_secret) == sol.GENERATED_SECRET_LENGTH) + self.assertTrue(len(sol.storage_secret) == secret_length) # assert id is hash of new secret self.assertTrue( secret_id_2 == hashlib.sha256(sol.storage_secret).hexdigest()) @@ -117,16 +119,18 @@ class SoledadSecretsTestCase(BaseSoledadTest): def test__has_secret(self): sol = self._soledad_instance( user='user@leap.se', prefix=self.rand_prefix) - self.assertTrue(sol._has_secret(), "Should have a secret at " - "this point") + self.assertTrue( + sol.secrets._has_secret(), + "Should have a secret at this point") # setting secret id to None should not interfere in the fact we have a # secret. - sol._set_secret_id(None) - self.assertTrue(sol._has_secret(), "Should have a secret at " - "this point") + sol.set_secret_id(None) + self.assertTrue( + sol.secrets._has_secret(), + "Should have a secret at this point") # but not being able to decrypt correctly should - sol._secrets[sol.secret_id][sol.SECRET_KEY] = None - self.assertFalse(sol._has_secret()) + sol.secrets._secrets[sol.secret_id][sol.secrets.SECRET_KEY] = None + self.assertFalse(sol.secrets._has_secret()) sol.close() diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index cb5348b4..acd0a54c 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -302,6 +302,7 @@ class EncryptedSyncTestCase( put_doc = mock.Mock(side_effect=_put_doc_side_effect) lock = mock.Mock(return_value=('atoken', 300)) unlock = mock.Mock() + close = mock.Mock() def __call__(self): return self @@ -373,9 +374,9 @@ class EncryptedSyncTestCase( sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') _, doclist = sol2.get_all_docs() self.assertEqual([], doclist) - sol2._secrets_path = sol1.secrets_path - sol2._load_secrets() - sol2._set_secret_id(sol1._secret_id) + sol2.secrets_path = sol1.secrets_path + sol2.secrets._load_secrets() + sol2.set_secret_id(sol1.secret_id) # sync the new instance sol2._server_url = self.getURL() sol2.sync() @@ -435,9 +436,9 @@ class EncryptedSyncTestCase( ) _, doclist = sol2.get_all_docs() self.assertEqual([], doclist) - sol2._secrets_path = sol1.secrets_path - sol2._load_secrets() - sol2._set_secret_id(sol1._secret_id) + sol2.secrets_path = sol1.secrets_path + sol2.secrets._load_secrets() + sol2.set_secret_id(sol1.secret_id) # sync the new instance sol2._server_url = self.getURL() sol2.sync() @@ -479,9 +480,9 @@ class EncryptedSyncTestCase( sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') _, doclist = sol2.get_all_docs() self.assertEqual([], doclist) - sol2._secrets_path = sol1.secrets_path - sol2._load_secrets() - sol2._set_secret_id(sol1._secret_id) + sol2.secrets_path = sol1.secrets_path + sol2.secrets._load_secrets() + sol2.set_secret_id(sol1.secret_id) # sync the new instance sol2._server_url = self.getURL() sol2.sync() @@ -524,9 +525,9 @@ class EncryptedSyncTestCase( sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') _, doclist = sol2.get_all_docs() self.assertEqual([], doclist) - sol2._secrets_path = sol1.secrets_path - sol2._load_secrets() - sol2._set_secret_id(sol1._secret_id) + sol2.secrets_path = sol1.secrets_path + sol2.secrets._load_secrets() + sol2.set_secret_id(sol1.secret_id) # sync the new instance sol2._server_url = self.getURL() sol2.sync() diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 11e43423..12bfbc3e 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -29,8 +29,9 @@ from leap.soledad.common.tests import ( from leap import soledad from leap.soledad.common.document import SoledadDocument from leap.soledad.common.crypto import WrongMac -from leap.soledad.client import Soledad, PassphraseTooShort -from leap.soledad.client.crypto import SoledadCrypto +from leap.soledad.client import Soledad +from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.secrets import PassphraseTooShort from leap.soledad.client.shared_db import SoledadSharedDatabase from leap.soledad.client.target import SoledadSyncTarget @@ -39,7 +40,6 @@ class AuxMethodsTestCase(BaseSoledadTest): def test__init_dirs(self): sol = self._soledad_instance(prefix='_init_dirs') - sol._init_dirs() local_db_dir = os.path.dirname(sol.local_db_path) secrets_path = os.path.dirname(sol.secrets_path) self.assertTrue(os.path.isdir(local_db_dir)) @@ -47,16 +47,9 @@ class AuxMethodsTestCase(BaseSoledadTest): sol.close() def test__init_db(self): - sol = self._soledad_instance() - sol._init_dirs() - sol._crypto = SoledadCrypto(sol) - #self._soledad._gpg.import_keys(PUBLIC_KEY) - if not sol._has_secret(): - sol._gen_secret() - sol._load_secrets() - sol._init_db() - from leap.soledad.client.sqlcipher import SQLCipherDatabase + sol = self._soledad_instance(prefix='_init_db') self.assertIsInstance(sol._db, SQLCipherDatabase) + self.assertTrue(os.path.isfile(sol.local_db_path)) sol.close() def test__init_config_defaults(self): @@ -71,16 +64,21 @@ class AuxMethodsTestCase(BaseSoledadTest): # instantiate without initializing so we just test _init_config() sol = SoledadMock() - Soledad._init_config(sol, None, None, '') + sol._passphrase = u'' + sol._secrets_path = None + sol._local_db_path = None + sol._server_url = '' + sol._init_config() # assert value of secrets_path self.assertEquals( os.path.join( sol.DEFAULT_PREFIX, Soledad.STORAGE_SECRETS_FILE_NAME), - sol.secrets_path) + sol._secrets_path) # assert value of local_db_path self.assertEquals( os.path.join(sol.DEFAULT_PREFIX, 'soledad.u1db'), sol.local_db_path) + sol.close() def test__init_config_from_params(self): """ @@ -174,8 +172,8 @@ class SoledadSharedDBTestCase(BaseSoledadTest): """ Ensure the shared db is queried with the correct doc_id. """ - doc_id = self._soledad._shared_db_doc_id() - self._soledad._get_secrets_from_shared_db() + doc_id = self._soledad.secrets._shared_db_doc_id() + self._soledad.secrets._get_secrets_from_shared_db() self.assertTrue( self._soledad._shared_db().get_doc.assert_called_with( doc_id) is None, @@ -185,8 +183,8 @@ class SoledadSharedDBTestCase(BaseSoledadTest): """ Ensure recovery document is put into shared recover db. """ - doc_id = self._soledad._shared_db_doc_id() - self._soledad._put_secrets_in_shared_db() + doc_id = self._soledad.secrets._shared_db_doc_id() + self._soledad.secrets._put_secrets_in_shared_db() self.assertTrue( self._soledad._shared_db().get_doc.assert_called_with( doc_id) is None, @@ -210,6 +208,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): def setUp(self): # mock signaling soledad.client.signal = Mock() + soledad.client.secrets.signal = Mock() # run parent's setUp BaseSoledadTest.setUp(self) @@ -231,57 +230,57 @@ class SoledadSignalingTestCase(BaseSoledadTest): - downloading keys / done downloading keys. - uploading keys / done uploading keys. """ - soledad.client.signal.reset_mock() + soledad.client.secrets.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( secrets_path='alternative_stage3.json', local_db_path='alternative_stage3.u1db') # reverse call order so we can verify in the order the signals were # expected - soledad.client.signal.mock_calls.reverse() - soledad.client.signal.call_args = \ - soledad.client.signal.call_args_list[0] - soledad.client.signal.call_args_list.reverse() + soledad.client.secrets.signal.mock_calls.reverse() + soledad.client.secrets.signal.call_args = \ + soledad.client.secrets.signal.call_args_list[0] + soledad.client.secrets.signal.call_args_list.reverse() # downloading keys signals - soledad.client.signal.assert_called_with( + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.signal) + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) # creating keys signals - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.signal) + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_CREATING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.signal) + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_DONE_CREATING_KEYS, ADDRESS, ) # downloading once more (inside _put_keys_in_shared_db) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.signal) + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.signal) + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) # uploading keys signals - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.signal) + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_UPLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.signal) + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_DONE_UPLOADING_KEYS, ADDRESS, ) @@ -298,8 +297,8 @@ class SoledadSignalingTestCase(BaseSoledadTest): # get existing instance so we have access to keys sol = self._soledad_instance() # create a document with secrets - doc = SoledadDocument(doc_id=sol._shared_db_doc_id()) - doc.content = sol.export_recovery_document() + doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id()) + doc.content = sol.secrets._export_recovery_document() class Stage2MockSharedDB(object): @@ -313,7 +312,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): sol.close() # reset mock - soledad.client.signal.reset_mock() + soledad.client.secrets.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( secrets_path='alternative_stage2.json', @@ -321,17 +320,17 @@ class SoledadSignalingTestCase(BaseSoledadTest): shared_db_class=Stage2MockSharedDB) # reverse call order so we can verify in the order the signals were # expected - soledad.client.signal.mock_calls.reverse() - soledad.client.signal.call_args = \ - soledad.client.signal.call_args_list[0] - soledad.client.signal.call_args_list.reverse() + soledad.client.secrets.signal.mock_calls.reverse() + soledad.client.secrets.signal.call_args = \ + soledad.client.secrets.signal.call_args_list[0] + soledad.client.secrets.signal.call_args_list.reverse() # assert download keys signals - soledad.client.signal.assert_called_with( + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.signal) - soledad.client.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.signal) + soledad.client.secrets.signal.assert_called_with( proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py index 595966ec..273ac06e 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py @@ -24,8 +24,6 @@ import threading from pysqlcipher import dbapi2 -from StringIO import StringIO -from urlparse import urljoin # u1db stuff. @@ -79,6 +77,7 @@ class TestSQLCipherBackendImpl(tests.TestCase): self.assertEqual(34, len(doc_id1)) int(doc_id1[len('D-'):], 16) self.assertNotEqual(doc_id1, db._allocate_doc_id()) + db.close() #----------------------------------------------------------------------------- @@ -123,9 +122,6 @@ class SQLCipherIndexTests(test_backends.DatabaseIndexTests): scenarios = SQLCIPHER_SCENARIOS -load_tests = tests.load_with_scenarios - - #----------------------------------------------------------------------------- # The following tests come from `u1db.tests.test_sqlite_backend`. #----------------------------------------------------------------------------- @@ -174,6 +170,8 @@ class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): self.assertIsInstance(outcome2[0], SQLCipherDatabaseTesting) db2 = outcome2[0] self.assertTrue(db2._is_initialized(db1._get_sqlite_handle().cursor())) + db1.close() + db2.close() class TestAlternativeDocument(SoledadDocument): @@ -190,22 +188,22 @@ class TestSQLCipherPartialExpandDatabase( def setUp(self): test_sqlite_backend.TestSQLitePartialExpandDatabase.setUp(self) self.db = SQLCipherDatabase(':memory:', PASSWORD) - self.db._set_replica_uid('test') + + def tearDown(self): + self.db.close() + test_sqlite_backend.TestSQLitePartialExpandDatabase.tearDown(self) def test_default_replica_uid(self): - self.db = SQLCipherDatabase(':memory:', PASSWORD) self.assertIsNot(None, self.db._replica_uid) self.assertEqual(32, len(self.db._replica_uid)) int(self.db._replica_uid, 16) def test__parse_index(self): - self.db = SQLCipherDatabase(':memory:', PASSWORD) g = self.db._parse_index_definition('fieldname') self.assertIsInstance(g, query_parser.ExtractField) self.assertEqual(['fieldname'], g.field) def test__update_indexes(self): - self.db = SQLCipherDatabase(':memory:', PASSWORD) g = self.db._parse_index_definition('fieldname') c = self.db._get_sqlite_handle().cursor() self.db._update_indexes('doc-id', {'fieldname': 'val'}, @@ -216,7 +214,6 @@ class TestSQLCipherPartialExpandDatabase( def test__set_replica_uid(self): # Start from scratch, so that replica_uid isn't set. - self.db = SQLCipherDatabase(':memory:', PASSWORD) self.assertIsNot(None, self.db._real_replica_uid) self.assertIsNot(None, self.db._replica_uid) self.db._set_replica_uid('foo') @@ -231,19 +228,23 @@ class TestSQLCipherPartialExpandDatabase( def test__open_database(self): temp_dir = self.createTempDir(prefix='u1db-test-') path = temp_dir + '/test.sqlite' - SQLCipherDatabase(path, PASSWORD) + db1 = SQLCipherDatabase(path, PASSWORD) db2 = SQLCipherDatabase._open_database(path, PASSWORD) self.assertIsInstance(db2, SQLCipherDatabase) + db1.close() + db2.close() def test__open_database_with_factory(self): temp_dir = self.createTempDir(prefix='u1db-test-') path = temp_dir + '/test.sqlite' - SQLCipherDatabase(path, PASSWORD) + db1 = SQLCipherDatabase(path, PASSWORD) db2 = SQLCipherDatabase._open_database( path, PASSWORD, document_factory=TestAlternativeDocument) doc = db2.create_doc({}) self.assertTrue(isinstance(doc, SoledadDocument)) + db1.close() + db2.close() def test__open_database_non_existent(self): temp_dir = self.createTempDir(prefix='u1db-test-') @@ -258,7 +259,9 @@ class TestSQLCipherPartialExpandDatabase( db = SQLCipherDatabase.__new__( SQLCipherDatabase) db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed + db._sync_db = None db._syncers = {} + db.sync_queue = None c = db._db_handle.cursor() c.execute('PRAGMA key="%s"' % PASSWORD) self.addCleanup(db.close) @@ -281,6 +284,8 @@ class TestSQLCipherPartialExpandDatabase( [None, SQLCipherDatabase._index_storage_value], observed) + db.close() + db2.close() def test__open_database_invalid(self): class SQLiteDatabaseTesting(SQLCipherDatabase): @@ -301,26 +306,32 @@ class TestSQLCipherPartialExpandDatabase( def test_open_database_existing(self): temp_dir = self.createTempDir(prefix='u1db-test-') path = temp_dir + '/existing.sqlite' - SQLCipherDatabase(path, PASSWORD) + db1 = SQLCipherDatabase(path, PASSWORD) db2 = SQLCipherDatabase.open_database(path, PASSWORD, create=False) self.assertIsInstance(db2, SQLCipherDatabase) + db1.close() + db2.close() def test_open_database_with_factory(self): temp_dir = self.createTempDir(prefix='u1db-test-') path = temp_dir + '/existing.sqlite' - SQLCipherDatabase(path, PASSWORD) + db1 = SQLCipherDatabase(path, PASSWORD) db2 = SQLCipherDatabase.open_database( path, PASSWORD, create=False, document_factory=TestAlternativeDocument) doc = db2.create_doc({}) self.assertTrue(isinstance(doc, SoledadDocument)) + db1.close() + db2.close() def test_open_database_create(self): temp_dir = self.createTempDir(prefix='u1db-test-') path = temp_dir + '/new.sqlite' - SQLCipherDatabase.open_database(path, PASSWORD, create=True) + db1 = SQLCipherDatabase.open_database(path, PASSWORD, create=True) db2 = SQLCipherDatabase.open_database(path, PASSWORD, create=False) self.assertIsInstance(db2, SQLCipherDatabase) + db1.close() + db2.close() def test_create_database_initializes_schema(self): # This test had to be cloned because our implementation of SQLCipher @@ -331,7 +342,8 @@ class TestSQLCipherPartialExpandDatabase( c = raw_db.cursor() c.execute("SELECT * FROM u1db_config") config = dict([(r[0], r[1]) for r in c.fetchall()]) - self.assertEqual({'sql_schema': '0', 'replica_uid': 'test', + replica_uid = self.db._replica_uid + self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, 'index_storage': 'expand referenced encrypted'}, config) @@ -444,6 +456,22 @@ class SQLCipherDatabaseSyncTests( def tearDown(self): test_sync.DatabaseSyncTests.tearDown(self) + if hasattr(self, 'db1') and isinstance(self.db1, SQLCipherDatabase): + self.db1.close() + if hasattr(self, 'db1_copy') \ + and isinstance(self.db1_copy, SQLCipherDatabase): + self.db1_copy.close() + if hasattr(self, 'db2') \ + and isinstance(self.db2, SQLCipherDatabase): + self.db2.close() + if hasattr(self, 'db2_copy') \ + and isinstance(self.db2_copy, SQLCipherDatabase): + self.db2_copy.close() + if hasattr(self, 'db3') \ + and isinstance(self.db3, SQLCipherDatabase): + self.db3.close() + + def test_sync_autoresolves(self): """ @@ -612,6 +640,9 @@ class SQLCipherDatabaseSyncTests( doc3.doc_id, doc3.rev, key, secret)) self.assertEqual(doc4.get_json(), doc3.get_json()) self.assertFalse(doc3.has_conflicts) + self.db1.close() + self.db2.close() + db3.close() def test_sync_puts_changes(self): """ @@ -778,6 +809,7 @@ class SQLCipherEncryptionTest(BaseLeapTest): doc = db.get_doc(doc.doc_id) self.assertEqual(tests.simple_doc, doc.get_json(), 'decrypted content mismatch') + db.close() def test_try_to_open_raw_db_with_sqlcipher_backend(self): """ @@ -790,7 +822,8 @@ class SQLCipherEncryptionTest(BaseLeapTest): try: # trying to open the a non-encrypted database with sqlcipher # backend should raise a DatabaseIsNotEncrypted exception. - SQLCipherDatabase(self.DB_FILE, PASSWORD) + db = SQLCipherDatabase(self.DB_FILE, PASSWORD) + db.close() raise dbapi2.DatabaseError( "SQLCipher backend should not be able to open non-encrypted " "dbs.") diff --git a/common/src/leap/soledad/common/tests/test_sync_deferred.py b/common/src/leap/soledad/common/tests/test_sync_deferred.py index 48e3150f..7643b27c 100644 --- a/common/src/leap/soledad/common/tests/test_sync_deferred.py +++ b/common/src/leap/soledad/common/tests/test_sync_deferred.py @@ -37,9 +37,6 @@ DEFER_DECRYPTION = True WAIT_STEP = 1 MAX_WAIT = 10 -from leap.soledad.common.tests import test_sqlcipher as ts -from leap.soledad.server import SoledadApp - from leap.soledad.client.sqlcipher import open as open_sqlcipher from leap.soledad.common.tests.util import SoledadWithCouchServerMixin @@ -89,11 +86,8 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): self._soledad.close() # XXX should not access "private" attrs - for f in [self._soledad._local_db_path, - self._soledad._secrets_path, - self.db1._sync_db_path]: - if os.path.isfile(f): - os.unlink(f) + import shutil + shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) #SQLCIPHER_SCENARIOS = [ diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py index edc4589b..45009f4e 100644 --- a/common/src/leap/soledad/common/tests/test_sync_target.py +++ b/common/src/leap/soledad/common/tests/test_sync_target.py @@ -23,29 +23,15 @@ import os import simplejson as json import u1db -from uuid import uuid4 - from u1db.remote import http_database -from u1db import SyncTarget -from u1db.sync import Synchronizer -from u1db.remote import ( - http_client, - http_database, - http_target, -) - -from leap.soledad import client from leap.soledad.client import ( target, auth, crypto, - VerifiedHTTPSConnection, sync, ) from leap.soledad.common.document import SoledadDocument -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests import BaseSoledadTest @@ -58,13 +44,6 @@ from leap.soledad.common.tests.util import ( from leap.soledad.common.tests.u1db_tests import test_backends from leap.soledad.common.tests.u1db_tests import test_remote_sync_target from leap.soledad.common.tests.u1db_tests import test_sync -from leap.soledad.common.tests.test_couch import ( - CouchDBTestCase, - CouchDBWrapper, -) - -from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware #----------------------------------------------------------------------------- @@ -279,8 +258,9 @@ class TestSoledadSyncTarget( def tearDown(self): SoledadWithCouchServerMixin.tearDown(self) tests.TestCaseWithServer.tearDown(self) - db, _ = self.request_state.ensure_database('test2') - db.delete_database() + db2, _ = self.request_state.ensure_database('test2') + db2.delete_database() + self.db1.close() def test_sync_exchange_send(self): """ @@ -540,6 +520,10 @@ class TestSoledadDbSync( self.main_test_class = test_sync.TestDbSync SoledadWithCouchServerMixin.setUp(self) + def tearDown(self): + SoledadWithCouchServerMixin.tearDown(self) + self.db.close() + def do_sync(self, target_name): """ Perform sync using SoledadSynchronizer, SoledadSyncTarget diff --git a/common/src/leap/soledad/common/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py index 6242099d..eb5e2874 100644 --- a/common/src/leap/soledad/common/tests/test_target.py +++ b/common/src/leap/soledad/common/tests/test_target.py @@ -22,17 +22,14 @@ Test Leap backend bits. import u1db import os -import ssl import simplejson as json import cStringIO -from u1db import SyncTarget from u1db.sync import Synchronizer from u1db.remote import ( http_client, http_database, - http_target, ) from leap.soledad import client @@ -40,7 +37,6 @@ from leap.soledad.client import ( target, auth, VerifiedHTTPSConnection, - sync, ) from leap.soledad.common.document import SoledadDocument from leap.soledad.server.auth import SoledadTokenAuthMiddleware @@ -61,10 +57,6 @@ from leap.soledad.common.tests.u1db_tests import test_document from leap.soledad.common.tests.u1db_tests import test_remote_sync_target from leap.soledad.common.tests.u1db_tests import test_https from leap.soledad.common.tests.u1db_tests import test_sync -from leap.soledad.common.tests.test_couch import ( - CouchDBTestCase, - CouchDBWrapper, -) #----------------------------------------------------------------------------- @@ -391,6 +383,10 @@ class TestSoledadSyncTarget( tests.TestCaseWithServer.tearDown(self) db, _ = self.request_state.ensure_database('test2') db.delete_database() + for i in ['db1', 'db2']: + if hasattr(self, i): + db = getattr(self, i) + db.close() def test_sync_exchange_send(self): """ @@ -413,6 +409,7 @@ class TestSoledadSyncTarget( self.assertEqual(1, new_gen) self.assertGetEncryptedDoc( db, 'doc-here', 'replica:1', '{"value": "here"}', False) + db.close() def test_sync_exchange_send_failure_and_retry_scenario(self): """ @@ -486,6 +483,7 @@ class TestSoledadSyncTarget( self.assertEqual( ('doc-here', 'replica:1', '{"value": "here"}', 1), other_changes[0][:-1]) + db.close() def test_sync_exchange_send_ensure_callback(self): """ @@ -515,6 +513,7 @@ class TestSoledadSyncTarget( self.assertEqual(db._replica_uid, replica_uid_box[0]) self.assertGetEncryptedDoc( db, 'doc-here', 'replica:1', '{"value": "here"}', False) + db.close() def test_sync_exchange_in_stream_error(self): # we bypass this test because our sync_exchange process does not @@ -747,6 +746,10 @@ class TestSoledadDbSync( self.main_test_class = test_sync.TestDbSync SoledadWithCouchServerMixin.setUp(self) + def tearDown(self): + SoledadWithCouchServerMixin.tearDown(self) + self.db.close() + def do_sync(self, target_name): """ Perform sync using SoledadSyncTarget and Token auth. diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py index 86e76fad..54adcde1 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py @@ -363,6 +363,7 @@ class LocalDatabaseTests(tests.DatabaseBaseTests): db2 = self.create_database('other-uid') doc2 = db2.create_doc_from_json(simple_doc) self.assertNotEqual(doc1.doc_id, doc2.doc_id) + db2.close() def test_put_doc_refuses_slashes_picky(self): doc = self.make_document('/a', None, simple_doc) -- cgit v1.2.3 From bb4ef28014b7846df8982f0008635f4d05b5a0b8 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 4 Aug 2014 10:34:20 -0300 Subject: Add instructions for closing SQLCipher db on docstrings. --- client/src/leap/soledad/client/sqlcipher.py | 37 ++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 85b0391b..7823e235 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -92,7 +92,16 @@ SQLITE_ISOLATION_LEVEL = None def open(path, password, create=True, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, defer_encryption=False): - """Open a database at the given location. + """ + Open a database at the given location. + + *** IMPORTANT *** + + Don't forget to close the database after use by calling the close() + method otherwise some resources might not be freed and you may experience + several kinds of leakages. + + *** IMPORTANT *** Will raise u1db.errors.DatabaseDoesNotExist if create=False and the database does not already exist. @@ -195,6 +204,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): Connect to an existing SQLCipher database, creating a new sqlcipher database file if needed. + *** IMPORTANT *** + + Don't forget to close the database after use by calling the close() + method otherwise some resources might not be freed and you may + experience several kinds of leakages. + + *** IMPORTANT *** + :param sqlcipher_file: The path for the SQLCipher file. :type sqlcipher_file: str :param password: The password that protects the SQLCipher db. @@ -356,6 +373,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ Open a SQLCipher database. + *** IMPORTANT *** + + Don't forget to close the database after use by calling the close() + method otherwise some resources might not be freed and you may + experience several kinds of leakages. + + *** IMPORTANT *** + :param sqlcipher_file: The path for the SQLCipher file. :type sqlcipher_file: str @@ -1097,6 +1122,16 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): del self.sync_queue self.sync_queue = None + def __del__(self): + """ + Free resources when deleting or garbage collecting the database. + + This is only here to minimze problems if someone ever forgets to call + the close() method after using the database; you should not rely on + garbage collecting to free up the database resources. + """ + self.close() + @property def replica_uid(self): return self._get_replica_uid() -- cgit v1.2.3 From 9f455ab44d8f229840a5c6a75e0e7b6a88b04f57 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 4 Aug 2014 11:40:37 -0300 Subject: Store decrypted storage secret in memory. --- client/src/leap/soledad/client/secrets.py | 182 +++++++++++++++++------------- 1 file changed, 105 insertions(+), 77 deletions(-) diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 3c6fc569..621e2d99 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -252,13 +252,9 @@ class SoledadSecrets(object): try: self._load_secrets() # try to load from disk except IOError as e: - logger.warning('IOError: %s' % str(e)) - try: - self.storage_secret - return True - except Exception as e: - logger.warning("Couldn't load storage secret: %s" % str(e)) - return False + logger.warning('IOError while loading secrets from disk: %s' % str(e)) + return False + return self.storage_secret is not None def _load_secrets(self): """ @@ -371,15 +367,21 @@ class SoledadSecrets(object): # create salt and key for calculating MAC salt = os.urandom(self.SALT_LENGTH) key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) + # encrypt secrets + encrypted_secrets = {} + for secret_id in self._secrets: + encrypted_secrets[secret_id] = self._encrypt_storage_secret( + self._secrets[secret_id]) + # create the recovery document data = { - self.STORAGE_SECRETS_KEY: self._secrets, + self.STORAGE_SECRETS_KEY: encrypted_secrets, self.KDF_KEY: self.KDF_SCRYPT, self.KDF_SALT_KEY: binascii.b2a_base64(salt), self.KDF_LENGTH_KEY: len(key), MAC_METHOD_KEY: MacMethods.HMAC, MAC_KEY: hmac.new( key, - json.dumps(self._secrets), + json.dumps(encrypted_secrets), sha256).hexdigest(), } return data @@ -425,10 +427,11 @@ class SoledadSecrets(object): 'contents.') # include secrets in the secret pool. secrets = 0 - for secret_id, secret_data in data[self.STORAGE_SECRETS_KEY].items(): + for secret_id, encrypted_secret in data[self.STORAGE_SECRETS_KEY].items(): if secret_id not in self._secrets: secrets += 1 - self._secrets[secret_id] = secret_data + self._secrets[secret_id] = \ + self._decrypt_storage_secret(encrypted_secret) return secrets, mac def _get_secrets_from_shared_db(self): @@ -480,30 +483,92 @@ class SoledadSecrets(object): # Management of secret for symmetric encryption. # - @property - def storage_secret(self): + def _decrypt_storage_secret(self, encrypted_secret_dict): """ - Return the storage secret. + Decrypt the storage secret. Storage secret is encrypted before being stored. This method decrypts - and returns the stored secret. + and returns the decrypted storage secret. - :return: The storage secret. + :param encrypted_secret_dict: The encrypted storage secret. + :type encrypted_secret_dict: dict + + :return: The decrypted storage secret. :rtype: str """ # calculate the encryption key + if encrypted_secret_dict[self.KDF_KEY] != self.KDF_SCRYPT: + raise Exception("Unknown KDF in stored secret.") key = scrypt.hash( self._passphrase_as_string(), # the salt is stored base64 encoded binascii.a2b_base64( - self._secrets[self._secret_id][self.KDF_SALT_KEY]), + encrypted_secret_dict[self.KDF_SALT_KEY]), buflen=32, # we need a key with 256 bits (32 bytes). ) + if encrypted_secret_dict[self.KDF_LENGTH_KEY] != len(key): + raise Exception("Wrong length of decryption key.") + if encrypted_secret_dict[self.CIPHER_KEY] != self.CIPHER_AES256: + raise Exception("Unknown cipher in stored secret.") # recover the initial value and ciphertext - iv, ciphertext = self._secrets[self._secret_id][self.SECRET_KEY].split( + iv, ciphertext = encrypted_secret_dict[self.SECRET_KEY].split( self.IV_SEPARATOR, 1) ciphertext = binascii.a2b_base64(ciphertext) - return self._crypto.decrypt_sym(ciphertext, key, iv=iv) + decrypted_secret = self._crypto.decrypt_sym(ciphertext, key, iv=iv) + if encrypted_secret_dict[self.LENGTH_KEY] != len(decrypted_secret): + raise Exception("Wrong length of decrypted secret.") + return decrypted_secret + + def _encrypt_storage_secret(self, decrypted_secret): + """ + Encrypt the storage secret. + + An encrypted secret has the following structure: + + { + '': { + 'kdf': 'scrypt', + 'kdf_salt': '' + 'kdf_length': + 'cipher': 'aes256', + 'length': , + 'secret': '', + } + } + + :param decrypted_secret: The decrypted storage secret. + :type decrypted_secret: str + + :return: The encrypted storage secret. + :rtype: dict + """ + # generate random salt + salt = os.urandom(self.SALT_LENGTH) + # get a 256-bit key + key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) + iv, ciphertext = self._crypto.encrypt_sym(decrypted_secret, key) + encrypted_secret_dict = { + # leap.soledad.crypto submodule uses AES256 for symmetric + # encryption. + self.KDF_KEY: self.KDF_SCRYPT, + self.KDF_SALT_KEY: binascii.b2a_base64(salt), + self.KDF_LENGTH_KEY: len(key), + self.CIPHER_KEY: self.CIPHER_AES256, + self.LENGTH_KEY: len(decrypted_secret), + self.SECRET_KEY: '%s%s%s' % ( + str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), + } + return encrypted_secret_dict + + @property + def storage_secret(self): + """ + Return the storage secret. + + :return: The decrypted storage secret. + :rtype: str + """ + return self._secrets.get(self._secret_id) def set_secret_id(self, secret_id): """ @@ -526,19 +591,6 @@ class SoledadSecrets(object): * SOLEDAD_CREATING_KEYS * SOLEDAD_DONE_CREATING_KEYS - A secret has the following structure: - - { - '': { - 'kdf': 'scrypt', - 'kdf_salt': '' - 'kdf_length': - 'cipher': 'aes256', - 'length': , - 'secret': '', - } - } - :return: The id of the generated secret. :rtype: str """ @@ -548,22 +600,7 @@ class SoledadSecrets(object): self.LOCAL_STORAGE_SECRET_LENGTH + self.REMOTE_STORAGE_SECRET_LENGTH) secret_id = sha256(secret).hexdigest() - # generate random salt - salt = os.urandom(self.SALT_LENGTH) - # get a 256-bit key - key = scrypt.hash(self._passphrase_as_string(), salt, buflen=32) - iv, ciphertext = self._crypto.encrypt_sym(secret, key) - self._secrets[secret_id] = { - # leap.soledad.crypto submodule uses AES256 for symmetric - # encryption. - self.KDF_KEY: self.KDF_SCRYPT, - self.KDF_SALT_KEY: binascii.b2a_base64(salt), - self.KDF_LENGTH_KEY: len(key), - self.CIPHER_KEY: self.CIPHER_AES256, - self.LENGTH_KEY: len(secret), - self.SECRET_KEY: '%s%s%s' % ( - str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), - } + self._secrets[secret_id] = secret self._store_secrets() signal(SOLEDAD_DONE_CREATING_KEYS, self._uuid) return secret_id @@ -596,24 +633,6 @@ class SoledadSecrets(object): # ensure there's a secret for which the passphrase will be changed. if not self._has_secret(): raise NoStorageSecret() - secret = self.storage_secret - # generate random salt - new_salt = os.urandom(self.SALT_LENGTH) - # get a 256-bit key - key = scrypt.hash(new_passphrase.encode('utf-8'), new_salt, buflen=32) - iv, ciphertext = self._crypto.encrypt_sym(secret, key) - # XXX update all secrets in the dict - self._secrets[self._secret_id] = { - # leap.soledad.crypto submodule uses AES256 for symmetric - # encryption. - self.KDF_KEY: self.KDF_SCRYPT, # TODO: remove hard coded kdf - self.KDF_SALT_KEY: binascii.b2a_base64(new_salt), - self.KDF_LENGTH_KEY: len(key), - self.CIPHER_KEY: self.CIPHER_AES256, - self.LENGTH_KEY: len(secret), - self.SECRET_KEY: '%s%s%s' % ( - str(iv), self.IV_SEPARATOR, binascii.b2a_base64(ciphertext)), - } self._passphrase = new_passphrase self._store_secrets() self._put_secrets_in_shared_db() @@ -655,27 +674,36 @@ class SoledadSecrets(object): # TODO: implement. pass - def get_remote_secret(self): + def _get_remote_storage_secret(self): """ Return the secret for remote storage. """ # TODO: implement pass - def get_local_storage_key(self): + + def _get_local_storage_secret(self): """ - Return the local storage key derived from the local storage secret. + Return the local storage secret. + """ + pwd_start = self.REMOTE_STORAGE_SECRET_LENGTH + self.SALT_LENGTH + pwd_end = self.REMOTE_STORAGE_SECRET_LENGTH + self.LOCAL_STORAGE_SECRET_LENGTH + return self.storage_secret[pwd_start:pwd_end] + + def _get_local_storage_salt(self): + """ + Return the local storage salt. """ - # salt indexes salt_start = self.REMOTE_STORAGE_SECRET_LENGTH salt_end = salt_start + self.SALT_LENGTH - # password indexes - pwd_start = salt_end - pwd_end = salt_start + self.LOCAL_STORAGE_SECRET_LENGTH - # calculate the key for local encryption - secret = self.storage_secret + return self.storage_secret[salt_start:salt_end] + + def get_local_storage_key(self): + """ + Return the local storage key derived from the local storage secret. + """ return scrypt.hash( - secret[pwd_start:pwd_end], # the password - secret[salt_start:salt_end], # the salt + self._get_local_storage_secret(), # the password + self._get_local_storage_salt(), # the salt buflen=32, # we need a key with 256 bits (32 bytes) ) -- cgit v1.2.3 From aa8fcba828bc917eaf8e6b0dacb76f0de904bf59 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 4 Aug 2014 16:17:09 -0300 Subject: Add salt for sync sb key derivation. --- client/src/leap/soledad/client/__init__.py | 7 ++ client/src/leap/soledad/client/crypto.py | 6 +- client/src/leap/soledad/client/secrets.py | 82 ++++++++++++++++++---- .../src/leap/soledad/common/tests/test_crypto.py | 20 +++--- 4 files changed, 88 insertions(+), 27 deletions(-) diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 0fd6672a..e66055e0 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -750,6 +750,13 @@ class Soledad(object): """ return self._secrets.storage_secret + @property + def remote_storage_secret(self): + """ + Return the secret used for encryption of remotelly stored data. + """ + return self._secrets.remote_storage_secret + @property def secrets(self): return self._secrets diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 4a64b5a8..1b01913d 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -240,9 +240,7 @@ class SoledadCrypto(object): if self.secret is None: raise NoSymmetricSecret() return hmac.new( - self.secret[ - MAC_KEY_LENGTH: - self._soledad.secrets.REMOTE_STORAGE_SECRET_LENGTH], + self.secret[MAC_KEY_LENGTH:], doc_id, hashlib.sha256).digest() @@ -251,7 +249,7 @@ class SoledadCrypto(object): # def _get_secret(self): - return self._soledad.storage_secret + return self._soledad.secrets.remote_storage_secret secret = property( _get_secret, doc='The secret used for symmetric encryption') diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 621e2d99..55580692 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -123,6 +123,14 @@ class SoledadSecrets(object): encryption. """ + GEN_SECRET_LENGTH = LOCAL_STORAGE_SECRET_LENGTH \ + + REMOTE_STORAGE_SECRET_LENGTH \ + + SALT_LENGTH # for sync db + """ + The length of the secret to be generated. This includes local and remote + secrets, and the salt for deriving the sync db secret. + """ + MINIMUM_PASSPHRASE_LENGTH = 6 """ The minimum length for a passphrase. The passphrase length is only checked @@ -268,12 +276,21 @@ class SoledadSecrets(object): with open(self._secrets_path, 'r') as f: content = json.loads(f.read()) _, mac = self._import_recovery_document(content) - if mac is False: - self._store_secrets() - self._put_secrets_in_shared_db() # choose first secret if no secret_id was given if self._secret_id is None: self.set_secret_id(self._secrets.items()[0][0]) + # enlarge secret if needed + enlarged = False + if len(self._secrets[self._secret_id]) < self.GEN_SECRET_LENGTH: + gen_len = self.GEN_SECRET_LENGTH \ + - len(self._secrets[self._secret_id]) + new_piece = os.urandom(gen_len) + self._secrets[self._secret_id] += new_piece + enlarged = True + # store and save in shared db if needed + if mac is False or enlarged is True: + self._store_secrets() + self._put_secrets_in_shared_db() def _get_or_gen_crypto_secrets(self): """ @@ -596,9 +613,7 @@ class SoledadSecrets(object): """ signal(SOLEDAD_CREATING_KEYS, self._uuid) # generate random secret - secret = os.urandom( - self.LOCAL_STORAGE_SECRET_LENGTH - + self.REMOTE_STORAGE_SECRET_LENGTH) + secret = os.urandom(self.GEN_SECRET_LENGTH) secret_id = sha256(secret).hexdigest() self._secrets[secret_id] = secret self._store_secrets() @@ -667,24 +682,29 @@ class SoledadSecrets(object): def _passphrase_as_string(self): return self._passphrase.encode('utf-8') - def get_syncdb_secret(self): - """ - Return the secret for sync db. - """ - # TODO: implement. - pass + # + # remote storage secret + # - def _get_remote_storage_secret(self): + @property + def remote_storage_secret(self): """ Return the secret for remote storage. """ - # TODO: implement - pass + key_start = 0 + key_end = self.REMOTE_STORAGE_SECRET_LENGTH + return self.storage_secret[key_start:key_end] + # + # local storage key + # def _get_local_storage_secret(self): """ Return the local storage secret. + + :return: The local storage secret. + :rtype: str """ pwd_start = self.REMOTE_STORAGE_SECRET_LENGTH + self.SALT_LENGTH pwd_end = self.REMOTE_STORAGE_SECRET_LENGTH + self.LOCAL_STORAGE_SECRET_LENGTH @@ -693,6 +713,9 @@ class SoledadSecrets(object): def _get_local_storage_salt(self): """ Return the local storage salt. + + :return: The local storage salt. + :rtype: str """ salt_start = self.REMOTE_STORAGE_SECRET_LENGTH salt_end = salt_start + self.SALT_LENGTH @@ -701,9 +724,38 @@ class SoledadSecrets(object): def get_local_storage_key(self): """ Return the local storage key derived from the local storage secret. + + :return: The key for protecting the local database. + :rtype: str """ return scrypt.hash( self._get_local_storage_secret(), # the password self._get_local_storage_salt(), # the salt buflen=32, # we need a key with 256 bits (32 bytes) ) + + # + # sync db key + # + + def _get_sync_db_salt(self): + """ + Return the salt for sync db. + """ + salt_start = self.LOCAL_STORAGE_SECRET_LENGTH \ + + self.REMOTE_STORAGE_SECRET_LENGTH + salt_end = salt_start + self.SALT_LENGTH + return self.storage_secret[salt_start:salt_end] + + def get_sync_db_key(self): + """ + Return the key for protecting the sync database. + + :return: The key for protecting the sync database. + :rtype: str + """ + return scrypt.hash( + self._get_local_storage_secret(), # the password + self._get_sync_db_salt(), # the salt + buflen=32, # we need a key with 256 bits (32 bytes) + ) diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index ccff5e46..0302a268 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -59,13 +59,18 @@ class RecoveryDocumentTestCase(BaseSoledadTest): def test_export_recovery_document_raw(self): rd = self._soledad.secrets._export_recovery_document() secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0] - secret = rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] + # assert exported secret is the same + secret = self._soledad.secrets._decrypt_storage_secret( + rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]) self.assertEqual(secret_id, self._soledad.secrets._secret_id) self.assertEqual(secret, self._soledad.secrets._secrets[secret_id]) - self.assertTrue(self._soledad.secrets.CIPHER_KEY in secret) - self.assertTrue(secret[self._soledad.secrets.CIPHER_KEY] == 'aes256') - self.assertTrue(self._soledad.secrets.LENGTH_KEY in secret) - self.assertTrue(self._soledad.secrets.SECRET_KEY in secret) + # assert recovery document structure + encrypted_secret = rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] + self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret) + self.assertTrue( + encrypted_secret[self._soledad.secrets.CIPHER_KEY] == 'aes256') + self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret) + self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) def test_import_recovery_document(self): rd = self._soledad.secrets._export_recovery_document() @@ -103,8 +108,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): # assert format of secret 1 self.assertTrue(sol.storage_secret is not None) self.assertIsInstance(sol.storage_secret, str) - secret_length = sol.secrets.LOCAL_STORAGE_SECRET_LENGTH \ - + sol.secrets.REMOTE_STORAGE_SECRET_LENGTH + secret_length = sol.secrets.GEN_SECRET_LENGTH self.assertTrue(len(sol.storage_secret) == secret_length) # assert format of secret 2 sol.set_secret_id(secret_id_2) @@ -129,7 +133,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): sol.secrets._has_secret(), "Should have a secret at this point") # but not being able to decrypt correctly should - sol.secrets._secrets[sol.secret_id][sol.secrets.SECRET_KEY] = None + sol.secrets._secrets[sol.secret_id] = None self.assertFalse(sol.secrets._has_secret()) sol.close() -- cgit v1.2.3 From 30aa5c040c093aa82be09e94dd403c18597320e5 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 4 Aug 2014 16:42:56 -0300 Subject: Protect sync db with a password. --- client/src/leap/soledad/client/__init__.py | 4 +++- client/src/leap/soledad/client/mp_safe_db.py | 15 +++++++++++++-- client/src/leap/soledad/client/sqlcipher.py | 28 ++++++++++++++++++---------- scripts/db_access/client_side_db.py | 3 ++- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index e66055e0..0b72be27 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -264,6 +264,7 @@ class Soledad(object): uses the 'raw PRAGMA key' format to handle the key to SQLCipher. """ key = self._secrets.get_local_storage_key() + sync_db_key = self._secrets.get_sync_db_key() self._db = sqlcipher_open( self._local_db_path, binascii.b2a_hex(key), # sqlcipher only accepts the hex version @@ -271,7 +272,8 @@ class Soledad(object): document_factory=SoledadDocument, crypto=self._crypto, raw_key=True, - defer_encryption=self._defer_encryption) + defer_encryption=self._defer_encryption, + sync_db_key=binascii.b2a_hex(sync_db_key)) def close(self): """ diff --git a/client/src/leap/soledad/client/mp_safe_db.py b/client/src/leap/soledad/client/mp_safe_db.py index a9ab5649..2c6b7e24 100644 --- a/client/src/leap/soledad/client/mp_safe_db.py +++ b/client/src/leap/soledad/client/mp_safe_db.py @@ -23,7 +23,7 @@ Multiprocessing-safe SQLite database. from threading import Thread from Queue import Queue -from sqlite3 import connect as sqlite3_connect +from pysqlcipher import dbapi2 # Thanks to http://code.activestate.com/recipes/526618/ @@ -49,7 +49,7 @@ class MPSafeSQLiteDB(Thread): """ Run the multiprocessing-safe database accessor. """ - conn = sqlite3_connect(self._db_path) + conn = dbapi2.connect(self._db_path) while True: req, arg, res = self._requests.get() if req == self.CLOSE: @@ -99,3 +99,14 @@ class MPSafeSQLiteDB(Thread): """ self.execute(self.CLOSE) self.join() + + def cursor(self): + """ + Return a fake cursor object. + + Not really a cursor, but allows for calling db.cursor().execute(). + + :return: Self. + :rtype: MPSafeSQLiteDatabase + """ + return self diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 7823e235..a7ddab24 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -91,7 +91,7 @@ SQLITE_ISOLATION_LEVEL = None def open(path, password, create=True, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, - cipher_page_size=1024, defer_encryption=False): + cipher_page_size=1024, defer_encryption=False, sync_db_key=None): """ Open a database at the given location. @@ -136,7 +136,8 @@ def open(path, password, create=True, document_factory=None, crypto=None, return SQLCipherDatabase.open_database( path, password, create=create, document_factory=document_factory, crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, - cipher_page_size=cipher_page_size, defer_encryption=defer_encryption) + cipher_page_size=cipher_page_size, defer_encryption=defer_encryption, + sync_db_key=sync_db_key) # @@ -199,7 +200,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): def __init__(self, sqlcipher_file, password, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', - kdf_iter=4000, cipher_page_size=1024): + kdf_iter=4000, cipher_page_size=1024, sync_db_key=None): """ Connect to an existing SQLCipher database, creating a new sqlcipher database file if needed. @@ -264,7 +265,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self._sync_db = None self._sync_db_write_lock = None self._sync_enc_pool = None - self._init_sync_db(sqlcipher_file) + self._init_sync_db(sqlcipher_file, sync_db_key) if self.defer_encryption: # initialize sync db @@ -293,7 +294,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): def _open_database(cls, sqlcipher_file, password, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, - defer_encryption=False): + defer_encryption=False, sync_db_key=None): """ Open a SQLCipher database. @@ -363,13 +364,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): return SQLCipherDatabase._sqlite_registry[v]( sqlcipher_file, password, document_factory=document_factory, crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, - cipher_page_size=cipher_page_size) + cipher_page_size=cipher_page_size, sync_db_key=sync_db_key) @classmethod def open_database(cls, sqlcipher_file, password, create, backend_cls=None, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, - cipher_page_size=1024, defer_encryption=False): + cipher_page_size=1024, defer_encryption=False, + sync_db_key=None): """ Open a SQLCipher database. @@ -429,7 +431,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): sqlcipher_file, password, document_factory=document_factory, crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, cipher_page_size=cipher_page_size, - defer_encryption=defer_encryption) + defer_encryption=defer_encryption, sync_db_key=sync_db_key) except u1db_errors.DatabaseDoesNotExist: if not create: raise @@ -440,7 +442,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): return backend_cls( sqlcipher_file, password, document_factory=document_factory, crypto=crypto, raw_key=raw_key, cipher=cipher, - kdf_iter=kdf_iter, cipher_page_size=cipher_page_size) + kdf_iter=kdf_iter, cipher_page_size=cipher_page_size, + sync_db_key=sync_db_key) def sync(self, url, creds=None, autocreate=True, defer_decryption=True): """ @@ -561,7 +564,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') - def _init_sync_db(self, sqlcipher_file): + def _init_sync_db(self, sqlcipher_file, sync_db_password): """ Initialize the Symmetrically-Encrypted document to be synced database, and the queue to communicate with subprocess workers. @@ -575,6 +578,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): else: sync_db_path = ":memory:" self._sync_db = MPSafeSQLiteDB(sync_db_path) + # protect the sync db with a password + if sync_db_password is not None: + self._set_crypto_pragmas( + self._sync_db, sync_db_password, True, + 'aes-256-cbc', 4000, 1024) self._sync_db_write_lock = threading.Lock() self._create_sync_db_tables() self.sync_queue = multiprocessing.Queue() diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 1c4f3754..67c5dbe1 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -120,7 +120,7 @@ def get_soledad_instance(username, provider, passphrase, basedir): server_url=server_url, cert_file=cert_file, auth_token=token, - defer_encryption=True) + defer_encryption=False) # main program @@ -154,3 +154,4 @@ if __name__ == '__main__': # get the soledad instance s = get_soledad_instance( args.username, args.provider, passphrase, basedir) + s.sync() -- cgit v1.2.3 From afdb1cefe605cabfe325df3124b9beb3174568ff Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 4 Aug 2014 16:48:21 -0300 Subject: Delete the received docs from sync db before starting a new sync. --- client/src/leap/soledad/client/crypto.py | 7 +++++++ client/src/leap/soledad/client/target.py | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 1b01913d..a24f2053 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -986,3 +986,10 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): else: # If no errors found, remove it from the received database. self.delete_received_doc(doc_id, doc_rev) + + def empty(self): + """ + Empty the received docs table of the sync database. + """ + sql = "DELETE FROM %s WHERE 1" % (self.TABLE_NAME,) + res = self._sync_db.execute(sql) diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 12175f48..1cb02856 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -1149,8 +1149,9 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): setProxiedObject(self._insert_doc_cb[source_replica_uid], return_doc_cb) + # empty the database before starting a new sync if defer_decryption is True and not self.clear_to_sync(): - raise PendingReceivedDocsSyncError + self._sync_decr_pool.empty() self._ensure_connection() if self._trace_hook: # for tests -- cgit v1.2.3 From ab7850bbdcded8b0e36cb27a2468f55d1910c218 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 12 Aug 2014 11:06:23 -0300 Subject: Fix bits from pullreq review. --- client/src/leap/soledad/client/__init__.py | 2 +- client/src/leap/soledad/client/crypto.py | 19 ++++-- client/src/leap/soledad/client/mp_safe_db.py | 2 +- client/src/leap/soledad/client/secrets.py | 69 +++++++++++++--------- client/src/leap/soledad/client/sqlcipher.py | 51 +++++++++------- client/src/leap/soledad/client/target.py | 7 +-- .../soledad/common/tests/test_sync_deferred.py | 3 +- 7 files changed, 92 insertions(+), 61 deletions(-) diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 0b72be27..c76e4a4a 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -755,7 +755,7 @@ class Soledad(object): @property def remote_storage_secret(self): """ - Return the secret used for encryption of remotelly stored data. + Return the secret used for encryption of remotely stored data. """ return self._secrets.remote_storage_secret diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index a24f2053..5e3760b3 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -863,7 +863,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :param encrypted: If not None, only return documents with encrypted field equal to given parameter. - :type encrypted: bool + :type encrypted: bool or None :return: list of doc_id, rev, generation, gen, trans_id :rtype: list @@ -878,16 +878,23 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): def get_insertable_docs_by_gen(self): """ - Return a list of documents ready to be inserted. + Return a list of non-encrypted documents ready to be inserted. """ + # here, we compare the list of all available docs with the list of + # decrypted docs and find the longest common prefix between these two + # lists. Note that the order of lists fetch matters: if instead we + # first fetch the list of decrypted docs and then the list of all + # docs, then some document might have been decrypted between these two + # calls, and if it is just the right doc then it might not be caught + # by the next loop. all_docs = self.get_docs_by_generation() decrypted_docs = self.get_docs_by_generation(encrypted=False) insertable = [] for doc_id, rev, _, gen, trans_id, encrypted in all_docs: try: - next_decrypted = decrypted_docs.next() - if doc_id == next_decrypted[0]: - content = next_decrypted[2] + next_doc_id, _, next_content, _, _, _ = decrypted_docs.next() + if doc_id == next_doc_id: + content = next_content insertable.append((doc_id, rev, content, gen, trans_id)) else: break @@ -901,7 +908,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :param encrypted: If not None, return count of documents with encrypted field equal to given parameter. - :type encrypted: bool + :type encrypted: bool or None :return: The count of documents. :rtype: int diff --git a/client/src/leap/soledad/client/mp_safe_db.py b/client/src/leap/soledad/client/mp_safe_db.py index 2c6b7e24..780b7153 100644 --- a/client/src/leap/soledad/client/mp_safe_db.py +++ b/client/src/leap/soledad/client/mp_safe_db.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# crypto.py +# mp_safe_db.py # Copyright (C) 2014 LEAP # # This program is free software: you can redistribute it and/or modify diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 55580692..b1c22371 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -69,21 +69,28 @@ logger = logging.getLogger(name=__name__) # Exceptions # -class NoStorageSecret(Exception): + +class SecretsException(Exception): + """ + Generic exception type raised by this module. + """ + + +class NoStorageSecret(SecretsException): """ Raised when trying to use a storage secret but none is available. """ pass -class PassphraseTooShort(Exception): +class PassphraseTooShort(SecretsException): """ Raised when trying to change the passphrase but the provided passphrase is too short. """ -class BootstrapSequenceError(Exception): +class BootstrapSequenceError(SecretsException): """ Raised when an attempt to generate a secret and store it in a recovery document on server failed. @@ -107,34 +114,35 @@ class SoledadSecrets(object): LOCAL_STORAGE_SECRET_LENGTH = 512 """ - The length of the secret used to derive a passphrase for the SQLCipher - database. + The length, in bytes, of the secret used to derive a passphrase for the + SQLCipher database. """ REMOTE_STORAGE_SECRET_LENGTH = 512 """ - The length of the secret used to derive an encryption key and a MAC auth - key for remote storage. + The length, in bytes, of the secret used to derive an encryption key and a + MAC auth key for remote storage. """ SALT_LENGTH = 64 """ - The length of the salt used to derive the key for the storage secret - encryption. + The length, in bytes, of the salt used to derive the key for the storage + secret encryption. """ GEN_SECRET_LENGTH = LOCAL_STORAGE_SECRET_LENGTH \ + REMOTE_STORAGE_SECRET_LENGTH \ + SALT_LENGTH # for sync db """ - The length of the secret to be generated. This includes local and remote - secrets, and the salt for deriving the sync db secret. + The length, in bytes, of the secret to be generated. This includes local + and remote secrets, and the salt for deriving the sync db secret. """ MINIMUM_PASSPHRASE_LENGTH = 6 """ - The minimum length for a passphrase. The passphrase length is only checked - when the user changes her passphrase, not when she instantiates Soledad. + The minimum length, in bytes, for a passphrase. The passphrase length is + only checked when the user changes her passphrase, not when she + instantiates Soledad. """ IV_SEPARATOR = ":" @@ -288,7 +296,7 @@ class SoledadSecrets(object): self._secrets[self._secret_id] += new_piece enlarged = True # store and save in shared db if needed - if mac is False or enlarged is True: + if not mac or enlarged: self._store_secrets() self._put_secrets_in_shared_db() @@ -443,13 +451,17 @@ class SoledadSecrets(object): raise WrongMac('Could not authenticate recovery document\'s ' 'contents.') # include secrets in the secret pool. - secrets = 0 + secret_count = 0 for secret_id, encrypted_secret in data[self.STORAGE_SECRETS_KEY].items(): if secret_id not in self._secrets: - secrets += 1 - self._secrets[secret_id] = \ - self._decrypt_storage_secret(encrypted_secret) - return secrets, mac + try: + self._secrets[secret_id] = \ + self._decrypt_storage_secret(encrypted_secret) + secret_count += 1 + except SecretsException as e: + logger.error("Failed to decrypt storage secret: %s" + % str(e)) + return secret_count, mac def _get_secrets_from_shared_db(self): """ @@ -512,10 +524,13 @@ class SoledadSecrets(object): :return: The decrypted storage secret. :rtype: str + + :raise SecretsException: Raised in case the decryption of the storage + secret fails for some reason. """ # calculate the encryption key if encrypted_secret_dict[self.KDF_KEY] != self.KDF_SCRYPT: - raise Exception("Unknown KDF in stored secret.") + raise SecretsException("Unknown KDF in stored secret.") key = scrypt.hash( self._passphrase_as_string(), # the salt is stored base64 encoded @@ -524,16 +539,16 @@ class SoledadSecrets(object): buflen=32, # we need a key with 256 bits (32 bytes). ) if encrypted_secret_dict[self.KDF_LENGTH_KEY] != len(key): - raise Exception("Wrong length of decryption key.") + raise SecretsException("Wrong length of decryption key.") if encrypted_secret_dict[self.CIPHER_KEY] != self.CIPHER_AES256: - raise Exception("Unknown cipher in stored secret.") + raise SecretsException("Unknown cipher in stored secret.") # recover the initial value and ciphertext iv, ciphertext = encrypted_secret_dict[self.SECRET_KEY].split( self.IV_SEPARATOR, 1) ciphertext = binascii.a2b_base64(ciphertext) decrypted_secret = self._crypto.decrypt_sym(ciphertext, key, iv=iv) if encrypted_secret_dict[self.LENGTH_KEY] != len(decrypted_secret): - raise Exception("Wrong length of decrypted secret.") + raise SecretsException("Wrong length of decrypted secret.") return decrypted_secret def _encrypt_storage_secret(self, decrypted_secret): @@ -729,8 +744,8 @@ class SoledadSecrets(object): :rtype: str """ return scrypt.hash( - self._get_local_storage_secret(), # the password - self._get_local_storage_salt(), # the salt + password=self._get_local_storage_secret(), + salt=self._get_local_storage_salt(), buflen=32, # we need a key with 256 bits (32 bytes) ) @@ -755,7 +770,7 @@ class SoledadSecrets(object): :rtype: str """ return scrypt.hash( - self._get_local_storage_secret(), # the password - self._get_sync_db_salt(), # the salt + password=self._get_local_storage_secret(), + salt=self._get_sync_db_salt(), buflen=32, # we need a key with 256 bits (32 bytes) ) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index a7ddab24..b7de2fba 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -63,6 +63,7 @@ from leap.soledad.client.target import SoledadSyncTarget from leap.soledad.client.target import PendingReceivedDocsSyncError from leap.soledad.client.sync import SoledadSynchronizer from leap.soledad.client.mp_safe_db import MPSafeSQLiteDB +from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument @@ -262,13 +263,16 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self._crypto = crypto # define sync-db attrs + self._sqlcipher_file = sqlcipher_file + self._sync_db_key = sync_db_key self._sync_db = None self._sync_db_write_lock = None self._sync_enc_pool = None - self._init_sync_db(sqlcipher_file, sync_db_key) + self.sync_queue = None if self.defer_encryption: # initialize sync db + self._init_sync_db() # initialize syncing queue encryption pool self._sync_enc_pool = SyncEncrypterPool( self._crypto, self._sync_db, self._sync_db_write_lock) @@ -471,6 +475,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): res = None # the following context manager blocks until the syncing lock can be # acquired. + if defer_decryption: + self._init_sync_db() with self.syncer(url, creds=creds) as syncer: # XXX could mark the critical section here... try: @@ -564,28 +570,27 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') - def _init_sync_db(self, sqlcipher_file, sync_db_password): + def _init_sync_db(self): """ Initialize the Symmetrically-Encrypted document to be synced database, and the queue to communicate with subprocess workers. - - :param sqlcipher_file: The path for the SQLCipher file. - :type sqlcipher_file: str """ - sync_db_path = None - if sqlcipher_file != ":memory:": - sync_db_path = "%s-sync" % sqlcipher_file - else: - sync_db_path = ":memory:" - self._sync_db = MPSafeSQLiteDB(sync_db_path) - # protect the sync db with a password - if sync_db_password is not None: - self._set_crypto_pragmas( - self._sync_db, sync_db_password, True, - 'aes-256-cbc', 4000, 1024) - self._sync_db_write_lock = threading.Lock() - self._create_sync_db_tables() - self.sync_queue = multiprocessing.Queue() + if self._sync_db is None: + soledad_assert(self._sync_db_key is not None) + sync_db_path = None + if self._sqlcipher_file != ":memory:": + sync_db_path = "%s-sync" % self._sqlcipher_file + else: + sync_db_path = ":memory:" + self._sync_db = MPSafeSQLiteDB(sync_db_path) + # protect the sync db with a password + if self._sync_db_key is not None: + self._set_crypto_pragmas( + self._sync_db, self._sync_db_key, False, + 'aes-256-cbc', 4000, 1024) + self._sync_db_write_lock = threading.Lock() + self._create_sync_db_tables() + self.sync_queue = multiprocessing.Queue() def _create_sync_db_tables(self): """ @@ -1106,24 +1111,30 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ Close db_handle and close syncer. """ - logger.debug("Sqlcipher backend: closing") + if logger is not None: # logger might be none if called from __del__ + logger.debug("Sqlcipher backend: closing") # stop the sync watcher for deferred encryption if self._sync_watcher is not None: self._sync_watcher.stop() self._sync_watcher.shutdown() + self._sync_watcher = None # close all open syncers for url in self._syncers: _, syncer = self._syncers[url] syncer.close() + self._syncers = [] # stop the encryption pool if self._sync_enc_pool is not None: self._sync_enc_pool.close() + self._sync_enc_pool = None # close the actual database if self._db_handle is not None: self._db_handle.close() + self._db_handle = None # close the sync database if self._sync_db is not None: self._sync_db.close() + self._sync_db = None # close the sync queue if self.sync_queue is not None: self.sync_queue.close() diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 1cb02856..ae2010a6 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -807,12 +807,9 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): self._sync_db = sync_db self._sync_db_write_lock = sync_db_write_lock - def _setup_sync_decr_pool(self, last_known_generation): + def _setup_sync_decr_pool(self): """ Set up the SyncDecrypterPool for deferred decryption. - - :param last_known_generation: Target's last known generation. - :type last_known_generation: int """ if self._sync_decr_pool is None: # initialize syncing queue decryption pool @@ -1133,7 +1130,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): if defer_decryption and self._sync_db is not None: self._sync_exchange_lock.acquire() - self._setup_sync_decr_pool(last_known_generation) + self._setup_sync_decr_pool() self._setup_sync_watcher() self._defer_decryption = True else: diff --git a/common/src/leap/soledad/common/tests/test_sync_deferred.py b/common/src/leap/soledad/common/tests/test_sync_deferred.py index 7643b27c..07a9742b 100644 --- a/common/src/leap/soledad/common/tests/test_sync_deferred.py +++ b/common/src/leap/soledad/common/tests/test_sync_deferred.py @@ -73,7 +73,8 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): self.db1 = open_sqlcipher(self.db1_file, DBPASS, create=True, document_factory=SoledadDocument, crypto=self._soledad._crypto, - defer_encryption=True) + defer_encryption=True, + sync_db_key=DBPASS) self.db2 = couch.CouchDatabase.open_database( urljoin( 'http://localhost:' + str(self.wrapper.port), 'test'), -- cgit v1.2.3 From cbf9ea06cebaf0a52ca958dd071c66952e70f1c8 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 19 Aug 2014 15:35:00 -0300 Subject: Include ddocs in sdist (#5896). --- common/MANIFEST.in | 1 + common/changes/bug_5896_include-ddocs-source-in-sdist | 2 ++ common/setup.py | 18 ------------------ 3 files changed, 3 insertions(+), 18 deletions(-) create mode 100644 common/changes/bug_5896_include-ddocs-source-in-sdist diff --git a/common/MANIFEST.in b/common/MANIFEST.in index 7f6148ef..8e5a2342 100644 --- a/common/MANIFEST.in +++ b/common/MANIFEST.in @@ -2,3 +2,4 @@ include pkg/* include versioneer.py include LICENSE include CHANGELOG +recursive-include src/leap/soledad/common/ddocs * diff --git a/common/changes/bug_5896_include-ddocs-source-in-sdist b/common/changes/bug_5896_include-ddocs-source-in-sdist new file mode 100644 index 00000000..94188f5e --- /dev/null +++ b/common/changes/bug_5896_include-ddocs-source-in-sdist @@ -0,0 +1,2 @@ + o Include couch design docs source files in source distribution and only + compile ddocs.py when building the package (#5896). diff --git a/common/setup.py b/common/setup.py index 6ee166ef..3650a15a 100644 --- a/common/setup.py +++ b/common/setup.py @@ -228,23 +228,6 @@ class cmd_develop(_cmd_develop): build_ddocs_py() -# versioneer powered -old_cmd_sdist = cmdclass["sdist"] - - -class cmd_sdist(old_cmd_sdist): - """ - Generate 'src/leap/soledad/common/ddocs.py' which contains couch design - documents scripts. - """ - def run(self): - old_cmd_sdist.run(self) - - def make_release_tree(self, base_dir, files): - old_cmd_sdist.make_release_tree(self, base_dir, files) - build_ddocs_py(basedir=base_dir) - - # versioneer powered old_cmd_build = cmdclass["build"] @@ -257,7 +240,6 @@ class cmd_build(old_cmd_build): cmdclass["freeze_debianver"] = freeze_debianver cmdclass["build"] = cmd_build -cmdclass["sdist"] = cmd_sdist cmdclass["develop"] = cmd_develop -- cgit v1.2.3 From a02fa40ccd5c328c666928c547f977565ecd75c1 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 27 Aug 2014 17:58:57 -0300 Subject: Do not depend on pysqlite2 (#2945). --- client/changes/VERSION_COMPAT | 11 +++++++++++ client/changes/bug_2945_do-not-depend-on-pysqlite2 | 1 + client/pkg/requirements.pip | 3 --- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 client/changes/bug_2945_do-not-depend-on-pysqlite2 diff --git a/client/changes/VERSION_COMPAT b/client/changes/VERSION_COMPAT index e69de29b..782844d7 100644 --- a/client/changes/VERSION_COMPAT +++ b/client/changes/VERSION_COMPAT @@ -0,0 +1,11 @@ +################################################# +# This file keeps track of the recent changes +# introduced in internal leap dependencies. +# Add your changes here so we can properly update +# requirements.pip during the release process. +# (leave header when resetting) +################################################# +# +# BEGIN DEPENDENCY LIST ------------------------- +# leap.foo.bar>=x.y.z +pysqlcipher>2.6.3 diff --git a/client/changes/bug_2945_do-not-depend-on-pysqlite2 b/client/changes/bug_2945_do-not-depend-on-pysqlite2 new file mode 100644 index 00000000..cf009a23 --- /dev/null +++ b/client/changes/bug_2945_do-not-depend-on-pysqlite2 @@ -0,0 +1 @@ + o Do not depend on pysqlite2 (#2945). diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index ae8d2dac..33286e4c 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -20,6 +20,3 @@ leap.soledad.common>=0.3.8 # this is not strictly needed by us, but we need it # until u1db adds it to its release as a dep. oauth - -# pysqlite should not be a dep, see #2945 -pysqlite -- cgit v1.2.3 From 2f1ee76a7169abc100efdf706f12a0abf6032f04 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 8 Sep 2014 17:12:45 -0300 Subject: Add benchmarking scripts. --- scripts/profiling/mail/__init__.py | 184 +++++++++++++++++ scripts/profiling/mail/couchdb.ini.template | 224 +++++++++++++++++++++ scripts/profiling/mail/couchdb_server.py | 42 ++++ scripts/profiling/mail/couchdb_wrapper.py | 84 ++++++++ .../5447A9AD50E3075ECCE432711B450E665FE63573.pub | 30 +++ .../5447A9AD50E3075ECCE432711B450E665FE63573.sec | 57 ++++++ scripts/profiling/mail/mail.py | 50 +++++ scripts/profiling/mail/mx.py | 80 ++++++++ scripts/profiling/mail/soledad_client.py | 40 ++++ scripts/profiling/mail/soledad_server.py | 48 +++++ scripts/profiling/mail/util.py | 8 + scripts/profiling/storage/benchmark-storage.py | 104 ++++++++++ .../profiling/storage/benchmark_storage_utils.py | 4 + scripts/profiling/storage/client_side_db.py | 1 + scripts/profiling/storage/plot.py | 94 +++++++++ scripts/profiling/storage/profile-format.py | 29 +++ scripts/profiling/storage/profile-storage.py | 107 ++++++++++ scripts/profiling/storage/util.py | 1 + scripts/profiling/sync/movingaverage.py | 1 + scripts/profiling/sync/profile-decoupled.py | 24 +++ 20 files changed, 1212 insertions(+) create mode 100644 scripts/profiling/mail/__init__.py create mode 100644 scripts/profiling/mail/couchdb.ini.template create mode 100644 scripts/profiling/mail/couchdb_server.py create mode 100644 scripts/profiling/mail/couchdb_wrapper.py create mode 100644 scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub create mode 100644 scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec create mode 100644 scripts/profiling/mail/mail.py create mode 100644 scripts/profiling/mail/mx.py create mode 100644 scripts/profiling/mail/soledad_client.py create mode 100644 scripts/profiling/mail/soledad_server.py create mode 100644 scripts/profiling/mail/util.py create mode 100644 scripts/profiling/storage/benchmark-storage.py create mode 100644 scripts/profiling/storage/benchmark_storage_utils.py create mode 120000 scripts/profiling/storage/client_side_db.py create mode 100755 scripts/profiling/storage/plot.py create mode 100644 scripts/profiling/storage/profile-format.py create mode 100755 scripts/profiling/storage/profile-storage.py create mode 120000 scripts/profiling/storage/util.py create mode 120000 scripts/profiling/sync/movingaverage.py create mode 100644 scripts/profiling/sync/profile-decoupled.py diff --git a/scripts/profiling/mail/__init__.py b/scripts/profiling/mail/__init__.py new file mode 100644 index 00000000..352faae6 --- /dev/null +++ b/scripts/profiling/mail/__init__.py @@ -0,0 +1,184 @@ +import threading +import time +import logging +import argparse + +from twisted.internet import reactor + +from util import log +from couchdb_server import get_couchdb_wrapper_and_u1db +from mx import put_lots_of_messages +from soledad_server import get_soledad_server +from soledad_client import SoledadClient +from mail import get_imap_server + + +UUID = 'blah' +AUTH_TOKEN = 'bleh' + + +logging.basicConfig(level=logging.DEBUG) + +modules = [ + 'gnupg', + 'leap.common', + 'leap.keymanager', + 'taskthread', +] + +for module in modules: + logger = logging.getLogger(name=module) + logger.setLevel(logging.WARNING) + + +class TestWatcher(threading.Thread): + + def __init__(self, couchdb_wrapper, couchdb_u1db, soledad_server, + soledad_client, imap_service, number_of_msgs, lock): + threading.Thread.__init__(self) + self._couchdb_wrapper = couchdb_wrapper + self._couchdb_u1db = couchdb_u1db + self._soledad_server = soledad_server + self._soledad_client = soledad_client + self._imap_service = imap_service + self._number_of_msgs = number_of_msgs + self._lock = lock + self._mails_available_time = None + self._mails_available_time_lock = threading.Lock() + self._conditions = None + + def run(self): + self._set_conditions() + while not self._test_finished(): + time.sleep(5) + log("TestWatcher: Tests finished, cleaning up...", + line_break=False) + self._stop_reactor() + self._cleanup() + log("done.") + self._lock.release() + + def _set_conditions(self): + self._conditions = [] + + # condition 1: number of received messages is equal to number of + # expected messages + def _condition1(*args): + msgcount = self._imap_service._inbox.getMessageCount() + cond = msgcount == self._number_of_msgs + log("[condition 1] received messages: %d (expected: %d) :: %s" + % (msgcount, self._number_of_msgs, cond)) + if self.mails_available_time == None \ + and cond: + with self._mails_available_time_lock: + self._mails_available_time = time.time() + return cond + + + # condition 2: number of documents in server is equal to in client + def _condition2(client_docs, server_docs): + cond = client_docs == server_docs + log("[condition 2] number of documents: client %d; server %d :: %s" + % (client_docs, server_docs, cond)) + return cond + + # condition 3: number of documents bigger than 3 x number of msgs + def _condition3(client_docs, *args): + cond = client_docs > (2 * self._number_of_msgs) + log("[condition 3] documents (%d) > 2 * msgs (%d) :: %s" + % (client_docs, self._number_of_msgs, cond)) + return cond + + # condition 4: not syncing + def _condition4(*args): + cond = not self._soledad_client.instance.syncing + log("[condition 4] not syncing :: %s" % cond) + return cond + + self._conditions.append(_condition1) + self._conditions.append(_condition2) + self._conditions.append(_condition3) + self._conditions.append(_condition4) + + def _test_finished(self): + client_docs = self._get_soledad_client_number_of_docs() + server_docs = self._get_couchdb_number_of_docs() + return not bool(filter(lambda x: not x(client_docs, server_docs), + self._conditions)) + + def _stop_reactor(self): + reactor.stop() + + def _cleanup(self): + self._imap_service.stop() + self._soledad_client.close() + self._soledad_server.stop() + self._couchdb_wrapper.stop() + + def _get_soledad_client_number_of_docs(self): + c = self._soledad_client.instance._db._db_handle.cursor() + c.execute('SELECT COUNT(*) FROM document WHERE content IS NOT NULL') + row = c.fetchone() + return int(row[0]) + + def _get_couchdb_number_of_docs(self): + couchdb = self._couchdb_u1db._database + view = couchdb.view('_all_docs', include_docs=True) + return len(filter( + lambda r: '_attachments' in r.values()[1] + and 'u1db_content' in r.values()[1]['_attachments'], + view.rows)) + + @property + def mails_available_time(self): + with self._mails_available_time_lock: + return self._mails_available_time + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('number_of_msgs', help="The number of documents", + type=int) + parser.add_argument('report_file', help="The name of the report file", + type=str) + args = parser.parse_args() + + # start a couchdb server + couchdb_wrapper, couchdb_u1db = get_couchdb_wrapper_and_u1db( + UUID, AUTH_TOKEN) + + put_time = put_lots_of_messages(couchdb_u1db, args.number_of_msgs) + + soledad_server = get_soledad_server(couchdb_wrapper.port) + + soledad_client = SoledadClient( + uuid='blah', + server_url='http://127.0.0.1:%d' % soledad_server.port, + auth_token=AUTH_TOKEN) + + imap_service = get_imap_server( + soledad_client.instance, UUID, 'snowden@bitmask.net', AUTH_TOKEN) + + lock = threading.Lock() + lock.acquire() + test_watcher = TestWatcher( + couchdb_wrapper, couchdb_u1db, soledad_server, soledad_client, + imap_service, args.number_of_msgs, lock) + test_watcher.start() + + # reactor.run() will block until TestWatcher stops the reactor. + start_time = time.time() + reactor.run() + log("Reactor stopped.") + end_time = time.time() + lock.acquire() + mails_available_time = test_watcher.mails_available_time - start_time + sync_time = end_time - start_time + log("Total syncing time: %f" % sync_time) + log("# number_of_msgs put_time mails_available_time sync_time") + result = "%d %f %f %f" \ + % (args.number_of_msgs, put_time, mails_available_time, + sync_time) + log(result) + with open(args.report_file, 'a') as f: + f.write(result + "\n") diff --git a/scripts/profiling/mail/couchdb.ini.template b/scripts/profiling/mail/couchdb.ini.template new file mode 100644 index 00000000..1fc2205b --- /dev/null +++ b/scripts/profiling/mail/couchdb.ini.template @@ -0,0 +1,224 @@ +; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure. + +; Upgrading CouchDB will overwrite this file. + +[couchdb] +database_dir = %(tempdir)s/lib +view_index_dir = %(tempdir)s/lib +max_document_size = 4294967296 ; 4 GB +os_process_timeout = 120000 ; 120 seconds. for view and external servers. +max_dbs_open = 100 +delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned +uri_file = %(tempdir)s/lib/couch.uri +file_compression = snappy + +[database_compaction] +; larger buffer sizes can originate smaller files +doc_buffer_size = 524288 ; value in bytes +checkpoint_after = 5242880 ; checkpoint after every N bytes were written + +[view_compaction] +; larger buffer sizes can originate smaller files +keyvalue_buffer_size = 2097152 ; value in bytes + +[httpd] +port = 0 +bind_address = 127.0.0.1 +authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler} +default_handler = {couch_httpd_db, handle_request} +secure_rewrites = true +vhost_global_handlers = _utils, _uuids, _session, _oauth, _users +allow_jsonp = false +; Options for the MochiWeb HTTP server. +;server_options = [{backlog, 128}, {acceptor_pool_size, 16}] +; For more socket options, consult Erlang's module 'inet' man page. +;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}] +log_max_chunk_size = 1000000 + +[log] +file = %(tempdir)s/log/couch.log +level = info +include_sasl = true + +[couch_httpd_auth] +authentication_db = _users +authentication_redirect = /_utils/session.html +require_valid_user = false +timeout = 600 ; number of seconds before automatic logout +auth_cache_size = 50 ; size is number of cache entries +allow_persistent_cookies = false ; set to true to allow persistent cookies + +[couch_httpd_oauth] +; If set to 'true', oauth token and consumer secrets will be looked up +; in the authentication database (_users). These secrets are stored in +; a top level property named "oauth" in user documents. Example: +; { +; "_id": "org.couchdb.user:joe", +; "type": "user", +; "name": "joe", +; "password_sha": "fe95df1ca59a9b567bdca5cbaf8412abd6e06121", +; "salt": "4e170ffeb6f34daecfd814dfb4001a73" +; "roles": ["foo", "bar"], +; "oauth": { +; "consumer_keys": { +; "consumerKey1": "key1Secret", +; "consumerKey2": "key2Secret" +; }, +; "tokens": { +; "token1": "token1Secret", +; "token2": "token2Secret" +; } +; } +; } +use_users_db = false + +[query_servers] +; javascript = %(tempdir)s/server/main.js +javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js +coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js + + +; Changing reduce_limit to false will disable reduce_limit. +; If you think you're hitting reduce_limit with a "good" reduce function, +; please let us know on the mailing list so we can fine tune the heuristic. +[query_server_config] +reduce_limit = true +os_process_limit = 25 + +[daemons] +view_manager={couch_view, start_link, []} +external_manager={couch_external_manager, start_link, []} +query_servers={couch_query_servers, start_link, []} +vhosts={couch_httpd_vhost, start_link, []} +httpd={couch_httpd, start_link, []} +stats_aggregator={couch_stats_aggregator, start, []} +stats_collector={couch_stats_collector, start, []} +uuids={couch_uuids, start, []} +auth_cache={couch_auth_cache, start_link, []} +replication_manager={couch_replication_manager, start_link, []} +os_daemons={couch_os_daemons, start_link, []} +compaction_daemon={couch_compaction_daemon, start_link, []} + +[httpd_global_handlers] +/ = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>} + +_all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req} +_active_tasks = {couch_httpd_misc_handlers, handle_task_status_req} +_config = {couch_httpd_misc_handlers, handle_config_req} +_replicate = {couch_httpd_replicator, handle_req} +_uuids = {couch_httpd_misc_handlers, handle_uuids_req} +_restart = {couch_httpd_misc_handlers, handle_restart_req} +_stats = {couch_httpd_stats_handlers, handle_stats_req} +_log = {couch_httpd_misc_handlers, handle_log_req} +_session = {couch_httpd_auth, handle_session_req} +_oauth = {couch_httpd_oauth, handle_oauth_req} + +[httpd_db_handlers] +_view_cleanup = {couch_httpd_db, handle_view_cleanup_req} +_compact = {couch_httpd_db, handle_compact_req} +_design = {couch_httpd_db, handle_design_req} +_temp_view = {couch_httpd_view, handle_temp_view_req} +_changes = {couch_httpd_db, handle_changes_req} + +; The external module takes an optional argument allowing you to narrow it to a +; single script. Otherwise the script name is inferred from the first path section +; after _external's own path. +; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>} +; _external = {couch_httpd_external, handle_external_req} + +[httpd_design_handlers] +_view = {couch_httpd_view, handle_view_req} +_show = {couch_httpd_show, handle_doc_show_req} +_list = {couch_httpd_show, handle_view_list_req} +_info = {couch_httpd_db, handle_design_info_req} +_rewrite = {couch_httpd_rewrite, handle_rewrite_req} +_update = {couch_httpd_show, handle_doc_update_req} + +; enable external as an httpd handler, then link it with commands here. +; note, this api is still under consideration. +; [external] +; mykey = /path/to/mycommand + +; Here you can setup commands for CouchDB to manage +; while it is alive. It will attempt to keep each command +; alive if it exits. +; [os_daemons] +; some_daemon_name = /path/to/script -with args + + +[uuids] +; Known algorithms: +; random - 128 bits of random awesome +; All awesome, all the time. +; sequential - monotonically increasing ids with random increments +; First 26 hex characters are random. Last 6 increment in +; random amounts until an overflow occurs. On overflow, the +; random prefix is regenerated and the process starts over. +; utc_random - Time since Jan 1, 1970 UTC with microseconds +; First 14 characters are the time in hex. Last 18 are random. +algorithm = sequential + +[stats] +; rate is in milliseconds +rate = 1000 +; sample intervals are in seconds +samples = [0, 60, 300, 900] + +[attachments] +compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression +compressible_types = text/*, application/javascript, application/json, application/xml + +[replicator] +db = _replicator +; Maximum replicaton retry count can be a non-negative integer or "infinity". +max_replication_retry_count = 10 +; More worker processes can give higher network throughput but can also +; imply more disk and network IO. +worker_processes = 4 +; With lower batch sizes checkpoints are done more frequently. Lower batch sizes +; also reduce the total amount of used RAM memory. +worker_batch_size = 500 +; Maximum number of HTTP connections per replication. +http_connections = 20 +; HTTP connection timeout per replication. +; Even for very fast/reliable networks it might need to be increased if a remote +; database is too busy. +connection_timeout = 30000 +; If a request fails, the replicator will retry it up to N times. +retries_per_request = 10 +; Some socket options that might boost performance in some scenarios: +; {nodelay, boolean()} +; {sndbuf, integer()} +; {recbuf, integer()} +; {priority, integer()} +; See the `inet` Erlang module's man page for the full list of options. +socket_options = [{keepalive, true}, {nodelay, false}] +; Path to a file containing the user's certificate. +;cert_file = /full/path/to/server_cert.pem +; Path to file containing user's private PEM encoded key. +;key_file = /full/path/to/server_key.pem +; String containing the user's password. Only used if the private keyfile is password protected. +;password = somepassword +; Set to true to validate peer certificates. +verify_ssl_certificates = false +; File containing a list of peer trusted certificates (in the PEM format). +;ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt +; Maximum peer certificate depth (must be set even if certificate validation is off). +ssl_certificate_max_depth = 3 + +[compaction_daemon] +; The delay, in seconds, between each check for which database and view indexes +; need to be compacted. +check_interval = 300 +; If a database or view index file is smaller then this value (in bytes), +; compaction will not happen. Very small files always have a very high +; fragmentation therefore it's not worth to compact them. +min_file_size = 131072 + +[compactions] +; List of compaction rules for the compaction daemon. + + +;[admins] +;testuser = -hashed-f50a252c12615697c5ed24ec5cd56b05d66fe91e,b05471ba260132953930cf9f97f327f5 +; pass for above user is 'testpass' diff --git a/scripts/profiling/mail/couchdb_server.py b/scripts/profiling/mail/couchdb_server.py new file mode 100644 index 00000000..2cf0a3fd --- /dev/null +++ b/scripts/profiling/mail/couchdb_server.py @@ -0,0 +1,42 @@ +import hashlib +import couchdb + +from leap.soledad.common.couch import CouchDatabase + +from util import log +from couchdb_wrapper import CouchDBWrapper + + +def start_couchdb_wrapper(): + log("Starting couchdb... ", line_break=False) + couchdb_wrapper = CouchDBWrapper() + couchdb_wrapper.start() + log("couchdb started on port %d." % couchdb_wrapper.port) + return couchdb_wrapper + + +def get_u1db_database(dbname, port): + return CouchDatabase.open_database( + 'http://127.0.0.1:%d/%s' % (port, dbname), + True, + ensure_ddocs=True) + + +def create_tokens_database(port, uuid, token_value): + tokens_database = couchdb.Server( + 'http://127.0.0.1:%d' % port).create('tokens') + token = couchdb.Document() + token['_id'] = hashlib.sha512(token_value).hexdigest() + token['user_id'] = uuid + token['type'] = 'Token' + tokens_database.save(token) + + +def get_couchdb_wrapper_and_u1db(uuid, token_value): + couchdb_wrapper = start_couchdb_wrapper() + + couchdb_u1db = get_u1db_database('user-%s' % uuid, couchdb_wrapper.port) + get_u1db_database('shared', couchdb_wrapper.port) + create_tokens_database(couchdb_wrapper.port, uuid, token_value) + + return couchdb_wrapper, couchdb_u1db diff --git a/scripts/profiling/mail/couchdb_wrapper.py b/scripts/profiling/mail/couchdb_wrapper.py new file mode 100644 index 00000000..cad1205b --- /dev/null +++ b/scripts/profiling/mail/couchdb_wrapper.py @@ -0,0 +1,84 @@ +import re +import os +import tempfile +import subprocess +import time +import shutil + + +from leap.common.files import mkdir_p + + +class CouchDBWrapper(object): + """ + Wrapper for external CouchDB instance. + """ + + def start(self): + """ + Start a CouchDB instance for a test. + """ + self.tempdir = tempfile.mkdtemp(suffix='.couch.test') + + path = os.path.join(os.path.dirname(__file__), + 'couchdb.ini.template') + handle = open(path) + conf = handle.read() % { + 'tempdir': self.tempdir, + } + handle.close() + + confPath = os.path.join(self.tempdir, 'test.ini') + handle = open(confPath, 'w') + handle.write(conf) + handle.close() + + # create the dirs from the template + mkdir_p(os.path.join(self.tempdir, 'lib')) + mkdir_p(os.path.join(self.tempdir, 'log')) + args = ['couchdb', '-n', '-a', confPath] + null = open('/dev/null', 'w') + + self.process = subprocess.Popen( + args, env=None, stdout=null.fileno(), stderr=null.fileno(), + close_fds=True) + # find port + logPath = os.path.join(self.tempdir, 'log', 'couch.log') + while not os.path.exists(logPath): + if self.process.poll() is not None: + got_stdout, got_stderr = "", "" + if self.process.stdout is not None: + got_stdout = self.process.stdout.read() + + if self.process.stderr is not None: + got_stderr = self.process.stderr.read() + raise Exception(""" +couchdb exited with code %d. +stdout: +%s +stderr: +%s""" % ( + self.process.returncode, got_stdout, got_stderr)) + time.sleep(0.01) + while os.stat(logPath).st_size == 0: + time.sleep(0.01) + PORT_RE = re.compile( + 'Apache CouchDB has started on http://127.0.0.1:(?P\d+)') + + handle = open(logPath) + m = None + line = handle.readline() + while m is None: + m = PORT_RE.search(line) + line = handle.readline() + handle.close() + self.port = int(m.group('port')) + + def stop(self): + """ + Terminate the CouchDB instance. + """ + self.process.terminate() + self.process.communicate() + shutil.rmtree(self.tempdir) + diff --git a/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub new file mode 100644 index 00000000..fee53b6d --- /dev/null +++ b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub @@ -0,0 +1,30 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.12 (GNU/Linux) + +mQENBFQEwmABCADC4wYD3mFt8xJtl3gjxRPEGN+FcgvzxxECIhyjYCHszrJu3f65 +/nyruriYdQLGR4YmUdERIwsZ7AMkAM1NAXe7sMq/gRPCb4PwrE7pRKzPAmaLeJMQ +DC9CSCP+2gUmzeKHS71GkddcUI1HFr1AX9lLVW2ScvmSzOllenyUoFKRvz2uGkLG +r5pvKsxJUHl9enpHRZV/0X5Y6PCinb4+eN2/ZTdpAywOycU+L+zflA0SOTCtf+dg +8k839T30piuBulDLNeOX84YcyXTW7XeCeRTg/ryoFaYhbOGt68BwnP9xlpU62LW0 +8vzSZ0mLm4Ttz2uaALEoLmsa91nyLi9pLtrRABEBAAG0IEVkIFNub3dkZW4gPHNu +b3dkZW5AYml0bWFzay5uZXQ+iQE4BBMBAgAiBQJUBMJgAhsDBgsJCAcDAgYVCAIJ +CgsEFgIDAQIeAQIXgAAKCRAbRQ5mX+Y1cx4RCACzEiHpmknl+HnB3bHGcr8VZvU9 +hIoclVR/OBjWQFUynr66XmaMHMOLAVoZkIPnezWQe3gDY7QlFCNCfz8SC2++4WtB +aBzal9IREkVnQBdnWalxLRviNH+zoFQ0URunBAyH4QAJRUC5tWfNj4yI6BCFPwXL +o0CCISIN+VMRAnwjABQD840/TbcMHDqmJyk/vpPYPFQqQudN3eB2hphKUkZMistP +O9++ui6glso+MgsbIUdqgnblM3FSrbjfLKekC+MeunFr8qRjettdaVyFD4GLg2SH +/JpsjZKYoZStatpdJcrNjUMsGtXLxaCPl+VldNuOKIsA85TZJomMiaBDqG9YuQEN +BFQEwmABCACrYiPXyGWHvs/aFKM63y9l6Th/+SKfzeq+ksLUI6fJIQytGORiiYZC +1LrhOTmir+dY3IygkFlldxehGt/OMUKLB774WhBDRI43rAhImwhNutTIuUTO7DsD +y7u83oVQH6xGZW5afs5BEU56Oa8DdUUA5gLfnpqAJG2mLB12JhClxzOYXK/VB0wJ +QsIWl+zyN7uLQr5xZOthzvP6p7MmsAjhzU1imwyEm8s91DLhwonuqadkMGKi2qHW +xuwxnr9aHQmobzy68/vOiBFeumr0YarirUdEDiUIti4rqy+0oteTNeMtXWo5rTtx +xeayw+TjjaOT2fZ6CAbq0I+lOW0aJrPFABEBAAGJAR8EGAECAAkFAlQEwmACGwwA +CgkQG0UOZl/mNXM0SggAuXzaLafCZiWx28K6mPKdgDOwTMm2rD7ukf3JiswlIyIU +/K19BENu82iHRSu4nb9amhHOLEhaf1Ep2JTf2Trmd+/SNh0kv3dSBNjCrvrMvtcA +qVxGc3DtRufGeRoy8ow/sEg+BCcfxJgR1efHOSQfMELDz2v8vbLbkR3Ubm7YRtKr +Ri2HWYrAXRrwFC07yqO2zptCND/LBtnMrp08AOSSLpRWVD/Ww6IE1v1UEN53aGsm +D+L/1XkuP4L9cqG3E2NYfsOPiblqRiKSe1adVid/rLn94u+fpE4kuvxoGKn1FJ/m +FqU8aPtxvPbsMkSoNOalxqJGpuWRTXTLb5I+Ed2Szw== +=yRE/ +-----END PGP PUBLIC KEY BLOCK----- diff --git a/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec new file mode 100644 index 00000000..64cb6c2a --- /dev/null +++ b/scripts/profiling/mail/keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec @@ -0,0 +1,57 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.12 (GNU/Linux) + +lQOYBFQEwmABCADC4wYD3mFt8xJtl3gjxRPEGN+FcgvzxxECIhyjYCHszrJu3f65 +/nyruriYdQLGR4YmUdERIwsZ7AMkAM1NAXe7sMq/gRPCb4PwrE7pRKzPAmaLeJMQ +DC9CSCP+2gUmzeKHS71GkddcUI1HFr1AX9lLVW2ScvmSzOllenyUoFKRvz2uGkLG +r5pvKsxJUHl9enpHRZV/0X5Y6PCinb4+eN2/ZTdpAywOycU+L+zflA0SOTCtf+dg +8k839T30piuBulDLNeOX84YcyXTW7XeCeRTg/ryoFaYhbOGt68BwnP9xlpU62LW0 +8vzSZ0mLm4Ttz2uaALEoLmsa91nyLi9pLtrRABEBAAEAB/0cLb885/amczMC7ZfN +dD17aS1ImkjoIqxu5ofFh6zgFLLwHOEr+4QDQKhYQvL3wHfBKqtUEwET6nA50HPe +4otxdAqczgkRYBZvwjpWuDtUY0B4giKhe2GJ7+xkeRmtlq9eaLEhdwzwqCUFVmBe +4n0Ey4FgX4d+lmpY5fEFfHjz4bZpoCrNZKtiGtOqdlKXm8PnU+ek+G7DFuavJ+g5 +B4fiqkLAYFX/IDFfaTSBYzNDPbSQR5n4Q4r9PdKazPXg7bnLuxAIY4i6KEXq2YpS +T1vLanCnBd4BEDUODCPZdc/AtbE0U+XoKTBjTvk3UEGIRJSsju8A1vWOG7UCl+0d +UMmRBADaiQYnp9QiwPDbpqxzlWN8Ms/+tAyRnBbhghcRqtrDSke6fSJAqXzVGVmF +FSJPMFf4mBYbr1U3YlYOJrlrb3tVhVN+7PTZDIaaENbtcsUAu7hTr7Ko6r1+WONC +yhtrtOR9sWHVbTZ09ZvyvjHnBqZVA2PuZLUn2wrimnIJbVNdlwQA5EwgoS8UuDob +hs6tLg29bAEDZRBHXQcDuEwdAX0KCHW0oQ0UE7exbDXXfQJSD9X3fDeqI+BdI+qQ +Yuauz+fJxKl+qHAcy5l5NT7qomEjHCzjGUnn4NJzkn6a3T4SrBdSMFY2hL/tJN0i +v1hXVNatjCEotqqsor+C6bf+Sl4I59cEAK+tYLTo/d+KOWtW4XbVhcYHjTBKtJGH +p2/wNb49ibYpkgOUqW2ebiCB0Lg6QEupomcaMOJGol3v8vwBKsuwQJhWJrAXC2sT +Bck5mI+DbabyAbYFtZgNHbcdDy62ADg1xD2Je7IjUDcpYaGB3VFhpD2rSvWDeSjR +3jTG3PPINfoBODK0IEVkIFNub3dkZW4gPHNub3dkZW5AYml0bWFzay5uZXQ+iQE4 +BBMBAgAiBQJUBMJgAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRAbRQ5m +X+Y1cx4RCACzEiHpmknl+HnB3bHGcr8VZvU9hIoclVR/OBjWQFUynr66XmaMHMOL +AVoZkIPnezWQe3gDY7QlFCNCfz8SC2++4WtBaBzal9IREkVnQBdnWalxLRviNH+z +oFQ0URunBAyH4QAJRUC5tWfNj4yI6BCFPwXLo0CCISIN+VMRAnwjABQD840/TbcM +HDqmJyk/vpPYPFQqQudN3eB2hphKUkZMistPO9++ui6glso+MgsbIUdqgnblM3FS +rbjfLKekC+MeunFr8qRjettdaVyFD4GLg2SH/JpsjZKYoZStatpdJcrNjUMsGtXL +xaCPl+VldNuOKIsA85TZJomMiaBDqG9YnQOYBFQEwmABCACrYiPXyGWHvs/aFKM6 +3y9l6Th/+SKfzeq+ksLUI6fJIQytGORiiYZC1LrhOTmir+dY3IygkFlldxehGt/O +MUKLB774WhBDRI43rAhImwhNutTIuUTO7DsDy7u83oVQH6xGZW5afs5BEU56Oa8D +dUUA5gLfnpqAJG2mLB12JhClxzOYXK/VB0wJQsIWl+zyN7uLQr5xZOthzvP6p7Mm +sAjhzU1imwyEm8s91DLhwonuqadkMGKi2qHWxuwxnr9aHQmobzy68/vOiBFeumr0 +YarirUdEDiUIti4rqy+0oteTNeMtXWo5rTtxxeayw+TjjaOT2fZ6CAbq0I+lOW0a +JrPFABEBAAEAB/4kyb13Z4MRyy37OkRakgdu2QvhfoVF59Hso/yxxFCTHibGLkpx +82LQTDEsQNgkGZ2vp7IBElM6MkDuemIRtOW7icdesJh+lAPyI9moWi0DYGgmCQzh +3PgDBdPQBDT6IL5eYw3323HjKjeeCW1NsPnFqlnyDe3MtWUbDyuozZ1ztA+Rekhb +UhEDK8ZccEKwpzrE2H5zBZLeY0OKKROGnwd1RBVXnHMgVRF7vbellYaR4h2odxOp +X8Ho4Xbs1h2VRNIuZwtfXxTIuTIfujlIPXMtVY40dgnEGt9PosJNr9IfGpfE3JCu +k9PTvq37aZkQbYj52nccwKdos+sLQgqAdHhZBADHg7B5jyRRObsCUXQ+jMHXxuqT +5l1twwOovvLC7YZoC8NAl4Bi0rh1Zj0ZEJJLFGzeiH+15C4qFTY+ospWpGu6X6g5 +I8ZWya8m2NSEWyJZNI1zKIU0iXucLevVTx+ctnovUNnb89v52/+BKr4k2iRISAzT +7RL63aFTgnLw9GKweQQA2+eU5jcQ6LobPY/fZZImnhwLDq/OaUV+7u1RfB04GA15 +HOGQV77np/QTM6b+ezKTFhG/HMCTqxf+HPHfzohBPF9zvboLvCkqaHBDiV9qYE96 +id/el3ZeWloLcEe62sMGbv0YYmsYWgJxL8BFGw5v1QpYbfQCnXLjyG+/9f6Ygq0D +/0W9X/NxWUyAXOv5KRy+rpkpNVxvie4tduvyVUa/9XHF7D/DMaXqkIvVX8yZUIDR +bjuIvGZkZ9QP8zf8NKkB98zbqZi6CbNrerjrDpb7Pj7uQd3GIcjW4UmENGA6t7U9 +IWen966PAXSzh3996tRHxwXexVIEdX5n4pO39ZiodEIOPzmJAR8EGAECAAkFAlQE +wmACGwwACgkQG0UOZl/mNXM0SggAuXzaLafCZiWx28K6mPKdgDOwTMm2rD7ukf3J +iswlIyIU/K19BENu82iHRSu4nb9amhHOLEhaf1Ep2JTf2Trmd+/SNh0kv3dSBNjC +rvrMvtcAqVxGc3DtRufGeRoy8ow/sEg+BCcfxJgR1efHOSQfMELDz2v8vbLbkR3U +bm7YRtKrRi2HWYrAXRrwFC07yqO2zptCND/LBtnMrp08AOSSLpRWVD/Ww6IE1v1U +EN53aGsmD+L/1XkuP4L9cqG3E2NYfsOPiblqRiKSe1adVid/rLn94u+fpE4kuvxo +GKn1FJ/mFqU8aPtxvPbsMkSoNOalxqJGpuWRTXTLb5I+Ed2Szw== +=9xZX +-----END PGP PRIVATE KEY BLOCK----- diff --git a/scripts/profiling/mail/mail.py b/scripts/profiling/mail/mail.py new file mode 100644 index 00000000..8504c762 --- /dev/null +++ b/scripts/profiling/mail/mail.py @@ -0,0 +1,50 @@ +import os +import threading + +from twisted.internet import reactor + +from leap.mail.imap.service import imap +from leap.keymanager import KeyManager + +from util import log + + +class IMAPServerThread(threading.Thread): + def __init__(self, imap_service): + threading.Thread.__init__(self) + self._imap_service = imap_service + + def run(self): + self._imap_service.start_loop() + reactor.run() + + def stop(self): + self._imap_service.stop() + reactor.stop() + + +def get_imap_server(soledad, uuid, address, token): + log("Starting imap... ", line_break=False) + + keymanager = KeyManager(address, '', soledad, token=token, uid=uuid) + with open( + os.path.join( + os.path.dirname(__file__), + 'keys/5447A9AD50E3075ECCE432711B450E665FE63573.sec'), 'r') as f: + pubkey, privkey = keymanager.parse_openpgp_ascii_key(f.read()) + keymanager.put_key(privkey) + + imap_service, imap_port, imap_factory = imap.run_service( + soledad, keymanager, userid=address, offline=False) + + imap_service.start_loop() + log("started.") + return imap_service + + #imap_server = IMAPServerThread(imap_service) + #try: + # imap_server.start() + #except Exception as e: + # print str(e) + + #return imap_server diff --git a/scripts/profiling/mail/mx.py b/scripts/profiling/mail/mx.py new file mode 100644 index 00000000..b6a1e5cf --- /dev/null +++ b/scripts/profiling/mail/mx.py @@ -0,0 +1,80 @@ +import datetime +import uuid +import json +import timeit + + +from leap.keymanager import openpgp +from leap.soledad.common.couch import CouchDocument +from leap.soledad.common.crypto import ( + EncryptionSchemes, + ENC_JSON_KEY, + ENC_SCHEME_KEY, +) + + +from util import log + + +message = """To: Ed Snowden +Date: %s +From: Glenn Greenwald + +hi! + +""" + + +def get_message(): + return message % datetime.datetime.now().strftime("%a %b %d %H:%M:%S:%f %Y") + + +def get_enc_json(pubkey, message): + with openpgp.TempGPGWrapper(gpgbinary='/usr/bin/gpg') as gpg: + gpg.import_keys(pubkey) + key = gpg.list_keys().pop() + # We don't care about the actual address, so we use a + # dummy one, we just care about the import of the pubkey + openpgp_key = openpgp._build_key_from_gpg("dummy@mail.com", + key, pubkey) + enc_json = str(gpg.encrypt( + json.dumps( + {'incoming': True, 'content': message}, + ensure_ascii=False), + openpgp_key.fingerprint, + symmetric=False)) + return enc_json + + +def get_new_doc(enc_json): + doc = CouchDocument(doc_id=str(uuid.uuid4())) + doc.content = { + 'incoming': True, + ENC_SCHEME_KEY: EncryptionSchemes.PUBKEY, + ENC_JSON_KEY: enc_json + } + return doc + + +def get_pubkey(): + with open('./keys/5447A9AD50E3075ECCE432711B450E665FE63573.pub') as f: + return f.read() + + +def put_one_message(pubkey, db): + enc_json = get_enc_json(pubkey, get_message()) + doc = get_new_doc(enc_json) + db.put_doc(doc) + + +def put_lots_of_messages(db, number): + log("Populating database with %d encrypted messages... " + % number, line_break=False) + pubkey = get_pubkey() + def _put_one_message(): + put_one_message(pubkey, db) + time = timeit.timeit(_put_one_message, number=number) + log("done.") + average_time = time / number + log("put_one_message average time: %f" % average_time) + return average_time diff --git a/scripts/profiling/mail/soledad_client.py b/scripts/profiling/mail/soledad_client.py new file mode 100644 index 00000000..5ac8ce39 --- /dev/null +++ b/scripts/profiling/mail/soledad_client.py @@ -0,0 +1,40 @@ +import tempfile +import os +import shutil + +from leap.soledad.client import Soledad + + +class SoledadClient(object): + + def __init__(self, uuid, server_url, auth_token): + self._uuid = uuid + self._server_url = server_url + self._auth_token = auth_token + self._tempdir = None + self._soledad = None + + @property + def instance(self): + if self._soledad is None: + self._soledad = self._get_soledad_client() + return self._soledad + + def _get_soledad_client(self): + self._tempdir = tempfile.mkdtemp() + return Soledad( + uuid=self._uuid, + passphrase=u'123', + secrets_path=os.path.join(self._tempdir, 'secrets.json'), + local_db_path=os.path.join(self._tempdir, 'soledad.db'), + server_url=self._server_url, + cert_file=None, + auth_token=self._auth_token, + secret_id=None, + defer_encryption=True) + + def close(self): + if self._soledad is not None: + self._soledad.close() + if self._tempdir is not None: + shutil.rmtree(self._tempdir) diff --git a/scripts/profiling/mail/soledad_server.py b/scripts/profiling/mail/soledad_server.py new file mode 100644 index 00000000..ad014456 --- /dev/null +++ b/scripts/profiling/mail/soledad_server.py @@ -0,0 +1,48 @@ +import threading + +from wsgiref.simple_server import make_server + +from leap.soledad.common.couch import CouchServerState + +from leap.soledad.server import SoledadApp +from leap.soledad.server.gzip_middleware import GzipMiddleware +from leap.soledad.server.auth import SoledadTokenAuthMiddleware + +from util import log + + +class SoledadServerThread(threading.Thread): + def __init__(self, server): + threading.Thread.__init__(self) + self._server = server + + def run(self): + self._server.serve_forever() + + def stop(self): + self._server.shutdown() + + @property + def port(self): + return self._server.server_port + + +def make_soledad_server_thread(couch_port): + state = CouchServerState( + 'http://127.0.0.1:%d' % couch_port, + 'shared', + 'tokens') + application = GzipMiddleware( + SoledadTokenAuthMiddleware(SoledadApp(state))) + server = make_server('', 0, application) + t = SoledadServerThread(server) + return t + + +def get_soledad_server(couchdb_port): + log("Starting soledad server... ", line_break=False) + soledad_server = make_soledad_server_thread(couchdb_port) + soledad_server.start() + log("soledad server started on port %d." % soledad_server.port) + return soledad_server + diff --git a/scripts/profiling/mail/util.py b/scripts/profiling/mail/util.py new file mode 100644 index 00000000..86118e88 --- /dev/null +++ b/scripts/profiling/mail/util.py @@ -0,0 +1,8 @@ +import sys + + +def log(msg, line_break=True): + sys.stdout.write(msg) + if line_break: + sys.stdout.write("\n") + sys.stdout.flush() diff --git a/scripts/profiling/storage/benchmark-storage.py b/scripts/profiling/storage/benchmark-storage.py new file mode 100644 index 00000000..79ee3270 --- /dev/null +++ b/scripts/profiling/storage/benchmark-storage.py @@ -0,0 +1,104 @@ +#!/usr/bin/python + +# scenarios: +# 1. soledad instantiation time. +# a. for unexisting db. +# b. for existing db. +# 2. soledad doc storage/retrieval. +# a. 1 KB document. +# b 10 KB. +# c. 100 KB. +# d. 1 MB. + + +import logging +import getpass +import tempfile +import argparse +import shutil +import timeit + + +from util import ValidateUserHandle + +# benchmarking args +REPEAT_NUMBER = 1000 +DOC_SIZE = 1024 + + +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + +def parse_args(): + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '-b', dest='basedir', required=False, default=None, + help='soledad base directory') + parser.add_argument( + '-p', dest='passphrase', required=False, default=None, + help='the user passphrase') + parser.add_argument( + '-l', dest='logfile', required=False, default='/tmp/benchhmark-storage.log', + help='the file to which write the benchmark logs') + args = parser.parse_args() + # get the password + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + # get the basedir + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + logger.info('Using %s as base directory.' % basedir) + + return args.username, args.provider, passphrase, basedir, args.logfile + + +if __name__ == '__main__': + username, provider, passphrase, basedir, logfile = parse_args() + create_results = [] + getall_results = [] + for i in [1, 200, 400, 600, 800, 1000]: + tempdir = tempfile.mkdtemp(dir=basedir) + setup_common = """ +import os +#from benchmark_storage_utils import benchmark_fun +#from benchmark_storage_utils import get_soledad_instance +from client_side_db import get_soledad_instance +sol = get_soledad_instance('%s', '%s', '%s', '%s') + """ % (username, provider, passphrase, tempdir) + + setup_create = setup_common + """ +content = {'data': os.urandom(%d/2).encode('hex')} +""" % (DOC_SIZE * i) + time = timeit.timeit( + 'sol.create_doc(content);', + setup=setup_create, number=REPEAT_NUMBER) + create_results.append((DOC_SIZE*i, time)) + print "CREATE: %d %f" % (DOC_SIZE*i, time) + + setup_get = setup_common + """ +doc_ids = [doc.doc_id for doc in sol.get_all_docs()[1]] +""" + + time = timeit.timeit( + "[sol.get_doc(doc_id) for doc_id in doc_ids]", + setup=setup_get, number=1) + getall_results.append((DOC_SIZE*i, time)) + print "GET_ALL: %d %f" % (DOC_SIZE*i, time) + shutil.rmtree(tempdir) + print "# size, time for creation of %d docs" % REPEAT_NUMBER + for size, time in create_results: + print size, time + print "# size, time for retrieval of %d docs" % REPEAT_NUMBER + for size, time in getall_results: + print size, time + shutil.rmtree(basedir) + diff --git a/scripts/profiling/storage/benchmark_storage_utils.py b/scripts/profiling/storage/benchmark_storage_utils.py new file mode 100644 index 00000000..fa8bb658 --- /dev/null +++ b/scripts/profiling/storage/benchmark_storage_utils.py @@ -0,0 +1,4 @@ +from client_side_db import get_soledad_instance + +def benchmark_fun(sol, content): + sol.create_doc(content) diff --git a/scripts/profiling/storage/client_side_db.py b/scripts/profiling/storage/client_side_db.py new file mode 120000 index 00000000..9e49a7f0 --- /dev/null +++ b/scripts/profiling/storage/client_side_db.py @@ -0,0 +1 @@ +../../db_access/client_side_db.py \ No newline at end of file diff --git a/scripts/profiling/storage/plot.py b/scripts/profiling/storage/plot.py new file mode 100755 index 00000000..280b9375 --- /dev/null +++ b/scripts/profiling/storage/plot.py @@ -0,0 +1,94 @@ +#!/usr/bin/python + + +# Create a plot of the results of running the ./benchmark-storage.py script. + + +import argparse +from matplotlib import pyplot as plt + +from sets import Set + + +def plot(filename, subtitle=''): + + # config the plot + plt.xlabel('doc size (KB)') + plt.ylabel('operation time (s)') + title = 'soledad 1000 docs creation/retrieval times' + if subtitle != '': + title += '- %s' % subtitle + plt.title(title) + + x = Set() + ycreate = [] + yget = [] + + ys = [] + #ys.append((ycreate, 'creation time', 'r', '-')) + #ys.append((yget, 'retrieval time', 'b', '-')) + + # read data from file + with open(filename, 'r') as f: + f.readline() + for i in xrange(6): + size, y = f.readline().strip().split(' ') + x.add(int(size)) + ycreate.append(float(y)) + + f.readline() + for i in xrange(6): + size, y = f.readline().strip().split(' ') + x.add(int(size)) + yget.append(float(y)) + + # get doc size in KB + x = list(x) + x.sort() + x = map(lambda val: val / 1024, x) + + # get normalized results per KB + nycreate = [] + nyget = [] + for i in xrange(len(x)): + nycreate.append(ycreate[i]/x[i]) + nyget.append(yget[i]/x[i]) + + ys.append((nycreate, 'creation time per KB', 'r', '-.')) + ys.append((nyget, 'retrieval time per KB', 'b', '-.')) + + for y in ys: + kwargs = { + 'linewidth': 1.0, + 'marker': '.', + 'color': y[2], + 'linestyle': y[3], + } + # normalize by doc size + plt.plot( + x, + y[0], + label=y[1], **kwargs) + + #plt.axes().get_xaxis().set_ticks(x) + #plt.axes().get_xaxis().set_ticklabels(x) + + # annotate max and min values + plt.xlim(0, 1100) + #plt.ylim(0, 350) + plt.grid() + plt.legend() + plt.show() + + +if __name__ == '__main__': + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'datafile', + help='the data file to plot') + parser.add_argument( + '-s', dest='subtitle', required=False, default='', + help='a subtitle for the plot') + args = parser.parse_args() + plot(args.datafile, args.subtitle) diff --git a/scripts/profiling/storage/profile-format.py b/scripts/profiling/storage/profile-format.py new file mode 100644 index 00000000..262a52ab --- /dev/null +++ b/scripts/profiling/storage/profile-format.py @@ -0,0 +1,29 @@ +#!/usr/bin/python + +import argparse +import pstats + + +def parse_args(): + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + '-f', dest='statsfiles', action='append', required=True, + help='a stats file') + args = parser.parse_args() + return args.statsfiles + + +def format_stats(statsfiles): + for f in statsfiles: + ps = pstats.Stats(f) + ps.strip_dirs() + ps.sort_stats('time') + ps.print_stats() + ps.sort_stats('cumulative') + ps.print_stats() + + +if __name__ == '__main__': + statsfiles = parse_args() + format_stats(statsfiles) diff --git a/scripts/profiling/storage/profile-storage.py b/scripts/profiling/storage/profile-storage.py new file mode 100755 index 00000000..305e6d5a --- /dev/null +++ b/scripts/profiling/storage/profile-storage.py @@ -0,0 +1,107 @@ +#!/usr/bin/python + +import os +import logging +import getpass +import tempfile +import argparse +import cProfile +import shutil +import pstats +import StringIO +import datetime + + +from client_side_db import get_soledad_instance +from util import ValidateUserHandle + +# profiling args +NUM_DOCS = 1 +DOC_SIZE = 1024**2 + + +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + +def parse_args(): + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '-b', dest='basedir', required=False, default=None, + help='soledad base directory') + parser.add_argument( + '-p', dest='passphrase', required=False, default=None, + help='the user passphrase') + parser.add_argument( + '-d', dest='logdir', required=False, default='/tmp/', + help='the direcroty to which write the profile stats') + args = parser.parse_args() + # get the password + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + # get the basedir + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + logger.info('Using %s as base directory.' % basedir) + + return args.username, args.provider, passphrase, basedir, args.logdir + +created_docs = [] + +def create_docs(sol, content): + for i in xrange(NUM_DOCS): + doc = sol.create_doc(content) + created_docs.append(doc.doc_id) + +def get_all_docs(sol): + for doc_id in created_docs: + sol.get_doc(doc_id) + +def do_profile(logdir, sol): + fname_prefix = os.path.join( + logdir, + "profile_%s" \ + % datetime.datetime.now().strftime('%Y-%m-%d_%H-%m-%S')) + + # profile create docs + content = {'data': os.urandom(DOC_SIZE/2).encode('hex')} + pr = cProfile.Profile() + pr.runcall( + create_docs, + sol, content) + s = StringIO.StringIO() + ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') + ps.print_stats() + ps.dump_stats("%s_creation.stats" % fname_prefix) + print s.getvalue() + + # profile get all docs + pr = cProfile.Profile() + pr.runcall( + get_all_docs, + sol) + s = StringIO.StringIO() + ps = pstats.Stats(pr, stream=s).sort_stats('cumulative') + ps.dump_stats("%s_retrieval.stats" % fname_prefix) + ps.print_stats() + print s.getvalue() + + +if __name__ == '__main__': + username, provider, passphrase, basedir, logdir = parse_args() + sol = get_soledad_instance( + username, + provider, + passphrase, + basedir) + do_profile(logdir, sol) + shutil.rmtree(basedir) + diff --git a/scripts/profiling/storage/util.py b/scripts/profiling/storage/util.py new file mode 120000 index 00000000..7f16d684 --- /dev/null +++ b/scripts/profiling/storage/util.py @@ -0,0 +1 @@ +../util.py \ No newline at end of file diff --git a/scripts/profiling/sync/movingaverage.py b/scripts/profiling/sync/movingaverage.py new file mode 120000 index 00000000..098b0a01 --- /dev/null +++ b/scripts/profiling/sync/movingaverage.py @@ -0,0 +1 @@ +../movingaverage.py \ No newline at end of file diff --git a/scripts/profiling/sync/profile-decoupled.py b/scripts/profiling/sync/profile-decoupled.py new file mode 100644 index 00000000..a844c3c6 --- /dev/null +++ b/scripts/profiling/sync/profile-decoupled.py @@ -0,0 +1,24 @@ +# test_name: soledad-sync +# start_time: 2014-06-12 20:09:11.232317+00:00 +# elapsed_time total_cpu total_memory proc_cpu proc_memory +0.000225 68.400000 46.100000 105.300000 0.527224 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.250167 0.000000 0.255160 +0.707006 76.200000 46.200000 90.000000 0.562369 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +1.413140 63.200000 46.100000 0.000000 0.360199 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +2.123962 0.000000 46.100000 0.000000 0.360199 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +2.833941 31.600000 46.100000 0.000000 0.360248 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +3.541532 5.300000 46.100000 0.000000 0.360298 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +4.253390 14.300000 46.100000 11.100000 0.360347 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +4.967365 5.000000 46.100000 0.000000 0.360347 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +5.680172 5.600000 46.100000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +6.390501 10.500000 46.100000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +7.101711 23.800000 46.000000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +7.810529 30.000000 46.000000 0.000000 0.360397 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +8.517835 25.000000 46.100000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +9.227455 5.300000 46.000000 9.500000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +9.936479 9.500000 46.000000 10.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +10.645015 52.400000 46.200000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +11.355179 21.100000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +12.066252 36.800000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +12.777689 28.600000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +13.489886 0.000000 46.000000 0.000000 0.361484 0.000000 0.255308 0.000000 0.250167 0.000000 0.250167 0.000000 0.255308 0.000000 0.255160 +# end_time: 2014-06-12 20:09:25.434677+00:00 \ No newline at end of file -- cgit v1.2.3 From 22d3a8d4c6a1e652109378245989f4f6a71d1f42 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 16 Sep 2014 11:41:31 -0500 Subject: comments + pep8 --- client/src/leap/soledad/client/crypto.py | 3 +++ client/src/leap/soledad/client/sync.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 5e3760b3..d68f3089 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -616,6 +616,9 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): :param content: The encrypted document. :type content: str """ + # FIXME --- callback should complete immediately since otherwise the + # thread which handles the results will get blocked + # Right now we're blocking the dispatcher with the writes to sqlite. sql_del = "DELETE FROM '%s' WHERE doc_id=?" % (self.TABLE_NAME,) sql_ins = "INSERT INTO '%s' VALUES (?, ?, ?)" % (self.TABLE_NAME,) diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index c158f2a7..0297c75c 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -120,7 +120,7 @@ class SoledadSynchronizer(Synchronizer): " target my gen: %d\n" " target my trans_id: %s" % (self.target_replica_uid, target_gen, target_trans_id, - target_my_gen, target_my_trans_id)) + target_my_gen, target_my_trans_id)) # make sure we'll have access to target replica uid once it exists if self.target_replica_uid is None: @@ -138,7 +138,7 @@ class SoledadSynchronizer(Synchronizer): # what's changed since that generation and this current gen my_gen, _, changes = self.source.whats_changed(target_my_gen) - logger.debug("Soledad sync: there are %d documents to send." \ + logger.debug("Soledad sync: there are %d documents to send." % len(changes)) # get source last-seen database generation for the target -- cgit v1.2.3 From 903ee560f26bec3ba9a7f01dcef3aaf0373515b4 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 26 Sep 2014 16:05:12 -0300 Subject: Clean and pep8 on couch.py. --- common/src/leap/soledad/common/couch.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/common/src/leap/soledad/common/couch.py b/common/src/leap/soledad/common/couch.py index 5658f4ce..a98a8f25 100644 --- a/common/src/leap/soledad/common/couch.py +++ b/common/src/leap/soledad/common/couch.py @@ -18,12 +18,12 @@ """A U1DB backend that uses CouchDB as its persistence layer.""" + import simplejson as json import re import uuid import logging import binascii -import socket import time import sys import threading @@ -44,7 +44,7 @@ from couchdb.http import ( urljoin as couch_urljoin, Resource, ) -from u1db import query_parser, vectorclock +from u1db import vectorclock from u1db.errors import ( DatabaseDoesNotExist, InvalidGeneration, @@ -60,7 +60,7 @@ from u1db.remote import http_app from u1db.remote.server_state import ServerState -from leap.soledad.common import USER_DB_PREFIX, ddocs, errors +from leap.soledad.common import ddocs, errors from leap.soledad.common.document import SoledadDocument @@ -160,7 +160,6 @@ class CouchDocument(SoledadDocument): """ if self._conflicts is None: raise Exception("Run self._ensure_fetch_conflicts first!") - conflicts_len = len(self._conflicts) self._conflicts = filter( lambda doc: doc.rev not in conflict_revs, self._conflicts) @@ -1181,7 +1180,7 @@ class CouchDatabase(CommonBackend): res = self._database.resource(*ddoc_path) try: with CouchDatabase.update_handler_lock[self._get_replica_uid()]: - body={ + body = { 'other_replica_uid': other_replica_uid, 'other_generation': other_generation, 'other_transaction_id': other_transaction_id, -- cgit v1.2.3 From 19f28c432f36022c5f1c0558f4742c864e7202c8 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 29 Sep 2014 11:19:16 -0300 Subject: Wait for last post request to finish before starting a new one during sync (#5975). --- client/changes/bug_5975_wait-for-last-request-on-sync | 1 + client/src/leap/soledad/client/target.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 client/changes/bug_5975_wait-for-last-request-on-sync diff --git a/client/changes/bug_5975_wait-for-last-request-on-sync b/client/changes/bug_5975_wait-for-last-request-on-sync new file mode 100644 index 00000000..7a394580 --- /dev/null +++ b/client/changes/bug_5975_wait-for-last-request-on-sync @@ -0,0 +1 @@ + o Wait for last post request to finish before starting a new one (#5975). diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index ae2010a6..651d3ee5 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -1176,6 +1176,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): synced = [] number_of_docs = len(docs_by_generations) + last_request_lock = None for doc, gen, trans_id in docs_by_generations: # allow for interrupting the sync process if self.stopped is True: @@ -1212,7 +1213,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): # end of symmetric encryption # ------------------------------------------------------------- t = syncer_pool.new_syncer_thread( - sent + 1, total, last_request_lock=None, + sent + 1, total, last_request_lock=last_request_lock, last_callback_lock=last_callback_lock) # bail out if any thread failed @@ -1249,7 +1250,12 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): # save thread and append t.start() threads.append((t, doc)) + + # update lock references so they can be used in next call to + # syncer_pool.new_syncer_thread() above last_callback_lock = t.callback_lock + last_request_lock = t.request_lock + sent += 1 # make sure all threads finished and we have up-to-date info -- cgit v1.2.3 From 1d7e51aad9e3cd649d0921b533669fa24cbd7ab2 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 7 Oct 2014 13:47:39 -0300 Subject: Bump version of dep on soledad.common. --- client/pkg/requirements.pip | 2 +- server/pkg/requirements.pip | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index 33286e4c..c694182d 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -11,7 +11,7 @@ zope.proxy # leap deps # -leap.soledad.common>=0.3.8 +leap.soledad.common>=0.6.0 # # XXX things to fix yet: diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index be5d156b..28717664 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -9,7 +9,7 @@ PyOpenSSL<0.14 twisted>=12.0.0 # leap deps -- bump me! -leap.soledad.common>=0.3.0 +leap.soledad.common>=0.6.0 # # Things yet to fix: -- cgit v1.2.3 From 9e43577e3068ad5718d3fa6fe28651c292e0ee6f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 8 Oct 2014 02:37:44 +0200 Subject: remove dep from README --- README.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.rst b/README.rst index fb909120..93922413 100644 --- a/README.rst +++ b/README.rst @@ -27,11 +27,6 @@ repository: :target: https://crate.io/packages/leap.soledad.server -Library dependencies --------------------- - -* ``libsqlite3-dev`` - Tests ----- -- cgit v1.2.3 From 5d8e1e4e210410c6f5702a4348fceba80ba03af6 Mon Sep 17 00:00:00 2001 From: Duda Dornelles Date: Thu, 27 Nov 2014 15:43:19 -0200 Subject: If the client loses and restores it connection we must reset the u1db sync_target connection for it to be able to sync again --- client/src/leap/soledad/client/sqlcipher.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index b7de2fba..26e74ef5 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -52,6 +52,7 @@ import json from hashlib import sha256 from contextlib import contextmanager from collections import defaultdict +from httplib import CannotSendRequest from pysqlcipher import dbapi2 from u1db.backends import sqlite_backend @@ -486,6 +487,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): except PendingReceivedDocsSyncError: logger.warning("Local sync db is not clear, skipping sync...") return + except CannotSendRequest: + logger.warning("Connection with sync target couldn't be established. Resetting connection...") + # closing the connection it will get it recreated in the next try + syncer.sync_target.close() + return return res -- cgit v1.2.3 From b51ea111366b207acfc4c78cbe2ed74188c3de1f Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 27 Nov 2014 16:56:00 -0200 Subject: Add changes file for #5855. --- client/changes/bug_5855_reset-connection-on-http-error-during-syn | 1 + 1 file changed, 1 insertion(+) create mode 100644 client/changes/bug_5855_reset-connection-on-http-error-during-syn diff --git a/client/changes/bug_5855_reset-connection-on-http-error-during-syn b/client/changes/bug_5855_reset-connection-on-http-error-during-syn new file mode 100644 index 00000000..e71a9a29 --- /dev/null +++ b/client/changes/bug_5855_reset-connection-on-http-error-during-syn @@ -0,0 +1 @@ + o Reset syncer connection when getting HTTP error during sync (#5855). -- cgit v1.2.3 From 3526d37350c27487fb1e4c6664dc346006ef72f4 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 27 Nov 2014 16:50:20 -0200 Subject: Fix pep8 style. --- client/src/leap/soledad/client/sqlcipher.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 26e74ef5..45629045 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -488,8 +488,9 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): logger.warning("Local sync db is not clear, skipping sync...") return except CannotSendRequest: - logger.warning("Connection with sync target couldn't be established. Resetting connection...") - # closing the connection it will get it recreated in the next try + logger.warning("Connection with sync target couldn't be " + "established. Resetting connection...") + # closing the connection it will be recreated in the next try syncer.sync_target.close() return -- cgit v1.2.3 From 4e90feb613da4f1f5221f3fed401d52dbf8f5e2b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 26 Nov 2014 21:06:25 +0100 Subject: force tls v1 in soledad client. Partially fixes #6437 --- client/changes/bug_6437_use_tls | 1 + client/src/leap/soledad/client/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 client/changes/bug_6437_use_tls diff --git a/client/changes/bug_6437_use_tls b/client/changes/bug_6437_use_tls new file mode 100644 index 00000000..7138d962 --- /dev/null +++ b/client/changes/bug_6437_use_tls @@ -0,0 +1 @@ + o Use TLS v1 in soledad client. Fixes partially #6437 diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index c76e4a4a..7267180b 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -811,7 +811,8 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection): self.sock = ssl.wrap_socket(sock, ca_certs=SOLEDAD_CERT, - cert_reqs=ssl.CERT_REQUIRED) + cert_reqs=ssl.CERT_REQUIRED, + ssl_version=ssl.PROTOCOL_TLSv1) match_hostname(self.sock.getpeercert(), self.host) -- cgit v1.2.3 From 17682563bd30e780cf7d620624a856376d257e83 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 26 Nov 2014 20:20:52 -0200 Subject: Enforce TLSv1 in soledad server (#6437). --- server/changes/bug_6437_avoid-sslv3 | 1 + server/pkg/soledad | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 server/changes/bug_6437_avoid-sslv3 diff --git a/server/changes/bug_6437_avoid-sslv3 b/server/changes/bug_6437_avoid-sslv3 new file mode 100644 index 00000000..5d41fbb3 --- /dev/null +++ b/server/changes/bug_6437_avoid-sslv3 @@ -0,0 +1 @@ + o Avoid use of SSLv3 (#6437). diff --git a/server/pkg/soledad b/server/pkg/soledad index 841233d1..62b7c5f8 100644 --- a/server/pkg/soledad +++ b/server/pkg/soledad @@ -19,6 +19,7 @@ CERT_PATH=/etc/leap/soledad-server.pem PRIVKEY_PATH=/etc/leap/soledad-server.key TWISTD_PATH=/usr/bin/twistd HOME=/var/lib/soledad/ +SSL_METHOD=TLSv1_METHOD [ -r /etc/default/soledad ] && . /etc/default/soledad @@ -35,7 +36,7 @@ case "$1" in --logfile=$LOGFILE \ web \ --wsgi=$OBJ \ - --port=ssl:$HTTPS_PORT:privateKey=$PRIVKEY_PATH:certKey=$CERT_PATH + --port=ssl:${HTTPS_PORT}:privateKey=${PRIVKEY_PATH}:certKey=${CERT_PATH}:sslmethod=${SSL_METHOD} echo "." ;; -- cgit v1.2.3 From 93bd3fb17670c0c8db5b50028ba2b3ce811dcf5d Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 26 Nov 2014 20:23:33 -0200 Subject: Run daemon as user soledad (#6436). --- server/changes/bug_6436_run-daemon-as-user-soledad | 1 + server/pkg/soledad | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 server/changes/bug_6436_run-daemon-as-user-soledad diff --git a/server/changes/bug_6436_run-daemon-as-user-soledad b/server/changes/bug_6436_run-daemon-as-user-soledad new file mode 100644 index 00000000..886964f1 --- /dev/null +++ b/server/changes/bug_6436_run-daemon-as-user-soledad @@ -0,0 +1 @@ + o Run daemon as user soledad (#6436). diff --git a/server/pkg/soledad b/server/pkg/soledad index 62b7c5f8..7f48e2c8 100644 --- a/server/pkg/soledad +++ b/server/pkg/soledad @@ -20,6 +20,8 @@ PRIVKEY_PATH=/etc/leap/soledad-server.key TWISTD_PATH=/usr/bin/twistd HOME=/var/lib/soledad/ SSL_METHOD=TLSv1_METHOD +USER=soledad +GROUP=soledad [ -r /etc/default/soledad ] && . /etc/default/soledad @@ -31,7 +33,9 @@ test -r /etc/leap/ || exit 0 case "$1" in start) echo -n "Starting soledad: twistd" - start-stop-daemon --start --quiet --exec $TWISTD_PATH -- \ + start-stop-daemon --start --quiet \ + --user=$USER --group=$GROUP \ + --exec $TWISTD_PATH -- \ --pidfile=$PIDFILE \ --logfile=$LOGFILE \ web \ -- cgit v1.2.3 From 2414b23ecdb8cfc8b8a5852243c22b6fbb89536f Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 28 Nov 2014 09:39:41 -0200 Subject: Enclose server initscript variables in curly brackets. --- server/pkg/soledad | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/pkg/soledad b/server/pkg/soledad index 7f48e2c8..bf24dac2 100644 --- a/server/pkg/soledad +++ b/server/pkg/soledad @@ -30,16 +30,16 @@ test -r /etc/leap/ || exit 0 . /lib/lsb/init-functions -case "$1" in +case "${1}" in start) echo -n "Starting soledad: twistd" start-stop-daemon --start --quiet \ - --user=$USER --group=$GROUP \ - --exec $TWISTD_PATH -- \ - --pidfile=$PIDFILE \ - --logfile=$LOGFILE \ + --user=${USER} --group=${GROUP} \ + --exec ${TWISTD_PATH} -- \ + --pidfile=${PIDFILE} \ + --logfile=${LOGFILE} \ web \ - --wsgi=$OBJ \ + --wsgi=${OBJ} \ --port=ssl:${HTTPS_PORT}:privateKey=${PRIVKEY_PATH}:certKey=${CERT_PATH}:sslmethod=${SSL_METHOD} echo "." ;; @@ -47,21 +47,21 @@ case "$1" in stop) echo -n "Stopping soledad: twistd" start-stop-daemon --stop --quiet \ - --pidfile $PIDFILE + --pidfile ${PIDFILE} echo "." ;; restart) - $0 stop - $0 start + ${0} stop + ${0} start ;; force-reload) - $0 restart + ${0} restart ;; status) - status_of_proc -p $PIDFILE $TWISTD_PATH soledad && exit 0 || exit $? + status_of_proc -p ${PIDFILE} ${TWISTD_PATH} soledad && exit 0 || exit ${?} ;; *) -- cgit v1.2.3 From 31eeafd715f407c61d8de4e6555241a1de33fba1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 3 Dec 2014 00:22:18 +0100 Subject: Use SSL negotiation. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although the API can be misleading, PROTOCOL_SSLv23 selects the highest protocol version that both the client and server support. Despite the name, this option can select “TLS” protocols as well as “SSL”. In this way, we can use TLSv1.2 (PROTOCOL_TLSv1 will *only* give us TLS v1.0) In the client side, we try to disable SSLv2 and SSLv3 options explicitely. The python version in wheezy does not offer PROTOCOL_TLSv1_2 nor OP_NO_SSLv2 or OP_NO_SSLv3 (It's new in 2.7.9) --- client/src/leap/soledad/client/__init__.py | 21 +++++++++++++++++---- server/pkg/soledad | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 7267180b..a4030d88 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -809,10 +809,23 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection): self.sock = sock self._tunnel() - self.sock = ssl.wrap_socket(sock, - ca_certs=SOLEDAD_CERT, - cert_reqs=ssl.CERT_REQUIRED, - ssl_version=ssl.PROTOCOL_TLSv1) + # negotiate the best availabe version... + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + + # but if possible, we want to disable bad ones + # needs python 2.7.9+ + try: + ctx.options |= ssl.OP_NO_SSLv2 + ctx.options |= ssl.OP_NO_SSLv3 + except AttributeError: + pass + + ctx.load_cert_chain(certfile=SOLEDAD_CERT) + ctx.verify_mode = ssl.CERT_REQUIRED + + self.sock = ctx.wrap_socket( + sock, server_side=True, server_hostname=self.host) + match_hostname(self.sock.getpeercert(), self.host) diff --git a/server/pkg/soledad b/server/pkg/soledad index bf24dac2..ccb3e9b0 100644 --- a/server/pkg/soledad +++ b/server/pkg/soledad @@ -19,7 +19,7 @@ CERT_PATH=/etc/leap/soledad-server.pem PRIVKEY_PATH=/etc/leap/soledad-server.key TWISTD_PATH=/usr/bin/twistd HOME=/var/lib/soledad/ -SSL_METHOD=TLSv1_METHOD +SSL_METHOD=SSLv23_METHOD USER=soledad GROUP=soledad -- cgit v1.2.3 From aafa79c0f5e3d05c28d8f41804ae692931e67d7e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 4 Dec 2014 18:13:06 +0100 Subject: fix ssl negotiation since ssl.SSLContext does not exist prior to python 2.7.9 --- client/src/leap/soledad/client/__init__.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index a4030d88..d7d01b57 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -809,22 +809,25 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection): self.sock = sock self._tunnel() - # negotiate the best availabe version... - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + highest_supported = ssl.PROTOCOL_SSLv23 - # but if possible, we want to disable bad ones - # needs python 2.7.9+ try: + # needs python 2.7.9+ + # negotiate the best available version, + # but explicitely disabled bad ones. + ctx = ssl.SSLContext(highest_supported) ctx.options |= ssl.OP_NO_SSLv2 ctx.options |= ssl.OP_NO_SSLv3 - except AttributeError: - pass - ctx.load_cert_chain(certfile=SOLEDAD_CERT) - ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_cert_chain(certfile=SOLEDAD_CERT) + ctx.verify_mode = ssl.CERT_REQUIRED + self.sock = ctx.wrap_socket( + sock, server_side=True, server_hostname=self.host) - self.sock = ctx.wrap_socket( - sock, server_side=True, server_hostname=self.host) + except AttributeError: + self.sock = ssl.wrap_socket( + sock, ca_certs=SOLEDAD_CERT, cert_reqs=ssl.CERT_REQUIRED, + ssl_version=highest_supported) match_hostname(self.sock.getpeercert(), self.host) -- cgit v1.2.3 From b761bfc3f95bc87461c8cc8ec8462b1a995ebddb Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 28 Nov 2014 11:41:25 -0200 Subject: Refactor client crypto for better code readability. --- client/src/leap/soledad/client/crypto.py | 233 ++++++++++++++++--------------- common/src/leap/soledad/common/crypto.py | 16 +++ 2 files changed, 136 insertions(+), 113 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index d68f3089..681bf4f7 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -33,59 +33,39 @@ from zope.proxy import sameProxiedObjects from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import EncryptionSchemes +from leap.soledad.common.crypto import EncryptionMethods +from leap.soledad.common.crypto import MacMethods +from leap.soledad.common.crypto import UnknownMacMethod +from leap.soledad.common.crypto import WrongMac +from leap.soledad.common.crypto import ENC_JSON_KEY +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.common.crypto import ENC_METHOD_KEY +from leap.soledad.common.crypto import ENC_IV_KEY +from leap.soledad.common.crypto import MAC_KEY +from leap.soledad.common.crypto import MAC_METHOD_KEY -from leap.soledad.common.crypto import ( - EncryptionSchemes, - UnknownEncryptionScheme, - MacMethods, - UnknownMacMethod, - WrongMac, - ENC_JSON_KEY, - ENC_SCHEME_KEY, - ENC_METHOD_KEY, - ENC_IV_KEY, - MAC_KEY, - MAC_METHOD_KEY, -) - logger = logging.getLogger(__name__) MAC_KEY_LENGTH = 64 -class EncryptionMethods(object): - """ - Representation of encryption methods that can be used. - """ - - AES_256_CTR = 'aes-256-ctr' - XSALSA20 = 'xsalsa20' - -# -# Exceptions -# - - -class DocumentNotEncrypted(Exception): - """ - Raised for failures in document encryption. +def _assert_known_encryption_method(method): """ - pass + Assert that we can encrypt/decrypt the given C{method} + :param method: The encryption method to assert. + :type method: str -class UnknownEncryptionMethod(Exception): - """ - Raised when trying to encrypt/decrypt with unknown method. - """ - pass - - -class NoSymmetricSecret(Exception): - """ - Raised when trying to get a hashed passphrase. + :raise AssertionError: Raised if C{method} is unknown. """ + valid_methods = [ + EncryptionMethods.AES_256_CTR, + EncryptionMethods.XSALSA20, + ] + soledad_assert(method in valid_methods) def encrypt_sym(data, key, method): @@ -104,13 +84,16 @@ def encrypt_sym(data, key, method): :return: A tuple with the initial value and the encrypted data. :rtype: (long, str) + + :raise AssertionError: Raised if C{method} is unknown. """ soledad_assert_type(key, str) - soledad_assert( len(key) == 32, # 32 x 8 = 256 bits. 'Wrong key size: %s bits (must be 256 bits long).' % (len(key) * 8)) + _assert_known_encryption_method(method) + iv = None # AES-256 in CTR mode if method == EncryptionMethods.AES_256_CTR: @@ -120,9 +103,7 @@ def encrypt_sym(data, key, method): elif method == EncryptionMethods.XSALSA20: iv = os.urandom(24) ciphertext = XSalsa20(key=key, iv=iv).process(data) - else: - # raise if method is unknown - raise UnknownEncryptionMethod('Unkwnown method: %s' % method) + return binascii.b2a_base64(iv), ciphertext @@ -143,6 +124,8 @@ def decrypt_sym(data, key, method, **kwargs): :return: The decrypted data. :rtype: str + + :raise AssertionError: Raised if C{method} is unknown. """ soledad_assert_type(key, str) # assert params @@ -152,6 +135,7 @@ def decrypt_sym(data, key, method, **kwargs): soledad_assert( 'iv' in kwargs, '%s needs an initial value.' % method) + _assert_known_encryption_method(method) # AES-256 in CTR mode if method == EncryptionMethods.AES_256_CTR: return AES( @@ -160,9 +144,6 @@ def decrypt_sym(data, key, method, **kwargs): return XSalsa20( key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data) - # raise if method is unknown - raise UnknownEncryptionMethod('Unkwnown method: %s' % method) - def doc_mac_key(doc_id, secret): """ @@ -176,17 +157,13 @@ def doc_mac_key(doc_id, secret): :param doc_id: The id of the document. :type doc_id: str - :param secret: soledad secret storage - :type secret: Soledad.storage_secret + :param secret: The Soledad storage secret + :type secret: str :return: The key. :rtype: str - - :raise NoSymmetricSecret: if no symmetric secret was supplied. """ - if secret is None: - raise NoSymmetricSecret() - + soledad_assert(secret is not None) return hmac.new( secret[:MAC_KEY_LENGTH], doc_id, @@ -234,11 +211,8 @@ class SoledadCrypto(object): :return: The passphrase. :rtype: str - - :raise NoSymmetricSecret: if no symmetric secret was supplied. """ - if self.secret is None: - raise NoSymmetricSecret() + soledad_assert(self.secret is not None) return hmac.new( self.secret[MAC_KEY_LENGTH:], doc_id, @@ -277,19 +251,25 @@ def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret): :type ciphertext: str :param mac_method: The MAC method to use. :type mac_method: str - :param secret: soledad secret - :type secret: Soledad.secret_storage + :param secret: The Soledad storage secret + :type secret: str :return: The calculated MAC. :rtype: str + + :raise UnknownMacMethod: Raised when C{mac_method} is unknown. """ - if mac_method == MacMethods.HMAC: - return hmac.new( - doc_mac_key(doc_id, secret), - str(doc_id) + str(doc_rev) + ciphertext, - hashlib.sha256).digest() - # raise if we do not know how to handle this MAC method - raise UnknownMacMethod('Unknown MAC method: %s.' % mac_method) + try: + soledad_assert(mac_method == MacMethods.HMAC) + except AssertionError: + raise UnknownMacMethod + content = str(doc_id) \ + + str(doc_rev) \ + + ciphertext + return hmac.new( + doc_mac_key(doc_id, secret), + content, + hashlib.sha256).digest() def encrypt_doc(crypto, doc): @@ -337,30 +317,37 @@ def encrypt_docstr(docstr, doc_id, doc_rev, key, secret): :param key: The key used to encrypt ``data`` (must be 256 bits long). :type key: str - :param secret: The Soledad secret (used for MAC auth). + :param secret: The Soledad storage secret (used for MAC auth). :type secret: str :return: The JSON serialization of the dict representing the encrypted content. :rtype: str """ - # encrypt content using AES-256 CTR mode + enc_scheme = EncryptionSchemes.SYMKEY + enc_method = EncryptionMethods.AES_256_CTR + mac_method = MacMethods.HMAC iv, ciphertext = encrypt_sym( str(docstr), # encryption/decryption routines expect str - key, method=EncryptionMethods.AES_256_CTR) + key, method=enc_method) + mac = binascii.b2a_hex( # store the mac as hex. + mac_doc( + doc_id, + doc_rev, + ciphertext, + mac_method, + secret)) # Return a representation for the encrypted content. In the following, we # convert binary data to hexadecimal representation so the JSON # serialization does not complain about what it tries to serialize. hex_ciphertext = binascii.b2a_hex(ciphertext) return json.dumps({ ENC_JSON_KEY: hex_ciphertext, - ENC_SCHEME_KEY: EncryptionSchemes.SYMKEY, - ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR, + ENC_SCHEME_KEY: enc_scheme, + ENC_METHOD_KEY: enc_method, ENC_IV_KEY: iv, - MAC_KEY: binascii.b2a_hex(mac_doc( # store the mac as hex. - doc_id, doc_rev, ciphertext, - MacMethods.HMAC, secret)), - MAC_METHOD_KEY: MacMethods.HMAC, + MAC_KEY: mac, + MAC_METHOD_KEY: mac_method, }) @@ -382,9 +369,47 @@ def decrypt_doc(crypto, doc): return decrypt_doc_dict(doc.content, doc.doc_id, doc.rev, key, secret) +def _verify_doc_mac(doc_id, doc_rev, ciphertext, mac_method, secret, doc_mac): + """ + Verify that C{doc_mac} is a correct MAC for the given document. + + :param doc_id: The id of the document. + :type doc_id: str + :param doc_rev: The revision of the document. + :type doc_rev: str + :param ciphertext: The content of the document. + :type ciphertext: str + :param mac_method: The MAC method to use. + :type mac_method: str + :param secret: The Soledad storage secret + :type secret: str + :param doc_mac: The MAC to be verified against. + :type doc_mac: str + + :raise UnknownMacMethod: Raised when C{mac_method} is unknown. + """ + calculated_mac = mac_doc( + doc_id, + doc_rev, + ciphertext, + mac_method, + secret) + # we compare mac's hashes to avoid possible timing attacks that might + # exploit python's builtin comparison operator behaviour, which fails + # immediatelly when non-matching bytes are found. + doc_mac_hash = hashlib.sha256( + binascii.a2b_hex( # the mac is stored as hex + doc_mac)).digest() + calculated_mac_hash = hashlib.sha256(calculated_mac).digest() + + if doc_mac_hash != calculated_mac_hash: + logger.warning("Wrong MAC while decrypting doc...") + raise WrongMac('Could not authenticate document\'s contents.') + + def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): """ - Decrypt C{doc}'s content. + Decrypt a symmetrically encrypted C{doc}'s content. Return the JSON string representation of the document's decrypted content. @@ -421,48 +446,30 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): :return: The JSON serialization of the decrypted content. :rtype: str + + :raise UnknownEncryptionMethod: Raised when trying to decrypt from an + unknown encryption method. """ + # assert document dictionary structure soledad_assert(ENC_JSON_KEY in doc_dict) soledad_assert(ENC_SCHEME_KEY in doc_dict) soledad_assert(ENC_METHOD_KEY in doc_dict) + soledad_assert(ENC_IV_KEY in doc_dict) soledad_assert(MAC_KEY in doc_dict) soledad_assert(MAC_METHOD_KEY in doc_dict) - # verify MAC - ciphertext = binascii.a2b_hex( # content is stored as hex. - doc_dict[ENC_JSON_KEY]) - mac = mac_doc( - doc_id, doc_rev, - ciphertext, - doc_dict[MAC_METHOD_KEY], secret) - # we compare mac's hashes to avoid possible timing attacks that might - # exploit python's builtin comparison operator behaviour, which fails - # immediatelly when non-matching bytes are found. - doc_mac_hash = hashlib.sha256( - binascii.a2b_hex( # the mac is stored as hex - doc_dict[MAC_KEY])).digest() - calculated_mac_hash = hashlib.sha256(mac).digest() - - if doc_mac_hash != calculated_mac_hash: - logger.warning("Wrong MAC while decrypting doc...") - raise WrongMac('Could not authenticate document\'s contents.') - # decrypt doc's content + ciphertext = binascii.a2b_hex(doc_dict[ENC_JSON_KEY]) enc_scheme = doc_dict[ENC_SCHEME_KEY] - plainjson = None - if enc_scheme == EncryptionSchemes.SYMKEY: - enc_method = doc_dict[ENC_METHOD_KEY] - if enc_method == EncryptionMethods.AES_256_CTR: - soledad_assert(ENC_IV_KEY in doc_dict) - plainjson = decrypt_sym( - ciphertext, key, - method=enc_method, - iv=doc_dict[ENC_IV_KEY]) - else: - raise UnknownEncryptionMethod(enc_method) - else: - raise UnknownEncryptionScheme(enc_scheme) + enc_method = doc_dict[ENC_METHOD_KEY] + enc_iv = doc_dict[ENC_IV_KEY] + doc_mac = doc_dict[MAC_KEY] + mac_method = doc_dict[MAC_METHOD_KEY] + + soledad_assert(enc_scheme == EncryptionSchemes.SYMKEY) + + _verify_doc_mac(doc_id, doc_rev, ciphertext, mac_method, secret, doc_mac) - return plainjson + return decrypt_sym(ciphertext, key, method=enc_method, iv=enc_iv) def is_symmetrically_encrypted(doc): @@ -540,7 +547,7 @@ def encrypt_doc_task(doc_id, doc_rev, content, key, secret): :type content: str :param key: The encryption key. :type key: str - :param secret: The Soledad secret (used for MAC auth). + :param secret: The Soledad storage secret (used for MAC auth). :type secret: str :return: A tuple containing the doc id, revision and encrypted content. @@ -646,7 +653,7 @@ def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret): :type trans_id: str :param key: The encryption key. :type key: str - :param secret: The Soledad secret (used for MAC auth). + :param secret: The Soledad storage secret (used for MAC auth). :type secret: str :return: A tuple containing the doc id, revision and encrypted content. diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py index 56bb608a..ab05999b 100644 --- a/common/src/leap/soledad/common/crypto.py +++ b/common/src/leap/soledad/common/crypto.py @@ -42,6 +42,22 @@ class UnknownEncryptionScheme(Exception): pass +class EncryptionMethods(object): + """ + Representation of encryption methods that can be used. + """ + + AES_256_CTR = 'aes-256-ctr' + XSALSA20 = 'xsalsa20' + + +class UnknownEncryptionMethod(Exception): + """ + Raised when trying to encrypt/decrypt with unknown method. + """ + pass + + class MacMethods(object): """ Representation of MAC methods used to authenticate document's contents. -- cgit v1.2.3 From ce0d421e41cfb75a3957541d6c88fcd7b26e8cd6 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 28 Nov 2014 11:50:15 -0200 Subject: Add encryption scheme, method and iv to symmetrically encrypted document MAC (#6400). --- .../feature_6400_include-iv-in-document-mac | 1 + client/src/leap/soledad/client/crypto.py | 174 ++++++++++++--------- client/src/leap/soledad/client/secrets.py | 79 ++++------ common/src/leap/soledad/common/crypto.py | 8 +- common/src/leap/soledad/common/tests/__init__.py | 2 +- .../src/leap/soledad/common/tests/test_crypto.py | 52 +++--- .../src/leap/soledad/common/tests/test_soledad.py | 62 ++++---- 7 files changed, 195 insertions(+), 183 deletions(-) create mode 100644 client/changes/feature_6400_include-iv-in-document-mac diff --git a/client/changes/feature_6400_include-iv-in-document-mac b/client/changes/feature_6400_include-iv-in-document-mac new file mode 100644 index 00000000..d8c9c9cc --- /dev/null +++ b/client/changes/feature_6400_include-iv-in-document-mac @@ -0,0 +1 @@ + o Include the IV in the encrypted document MAC (#6400). diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 681bf4f7..d6d9a618 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -32,18 +32,8 @@ from zope.proxy import sameProxiedObjects from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type +from leap.soledad.common import crypto from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.crypto import EncryptionSchemes -from leap.soledad.common.crypto import EncryptionMethods -from leap.soledad.common.crypto import MacMethods -from leap.soledad.common.crypto import UnknownMacMethod -from leap.soledad.common.crypto import WrongMac -from leap.soledad.common.crypto import ENC_JSON_KEY -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.common.crypto import ENC_METHOD_KEY -from leap.soledad.common.crypto import ENC_IV_KEY -from leap.soledad.common.crypto import MAC_KEY -from leap.soledad.common.crypto import MAC_METHOD_KEY logger = logging.getLogger(__name__) @@ -59,13 +49,16 @@ def _assert_known_encryption_method(method): :param method: The encryption method to assert. :type method: str - :raise AssertionError: Raised if C{method} is unknown. + :raise UnknownEncryptionMethodError: Raised when C{method} is unknown. """ valid_methods = [ - EncryptionMethods.AES_256_CTR, - EncryptionMethods.XSALSA20, + crypto.EncryptionMethods.AES_256_CTR, + crypto.EncryptionMethods.XSALSA20, ] - soledad_assert(method in valid_methods) + try: + soledad_assert(method in valid_methods) + except AssertionError: + raise crypto.UnknownEncryptionMethodError def encrypt_sym(data, key, method): @@ -96,11 +89,11 @@ def encrypt_sym(data, key, method): iv = None # AES-256 in CTR mode - if method == EncryptionMethods.AES_256_CTR: + if method == crypto.EncryptionMethods.AES_256_CTR: iv = os.urandom(16) ciphertext = AES(key=key, iv=iv).process(data) # XSalsa20 - elif method == EncryptionMethods.XSALSA20: + elif method == crypto.EncryptionMethods.XSALSA20: iv = os.urandom(24) ciphertext = XSalsa20(key=key, iv=iv).process(data) @@ -125,7 +118,7 @@ def decrypt_sym(data, key, method, **kwargs): :return: The decrypted data. :rtype: str - :raise AssertionError: Raised if C{method} is unknown. + :raise UnknownEncryptionMethodError: Raised when C{method} is unknown. """ soledad_assert_type(key, str) # assert params @@ -137,10 +130,10 @@ def decrypt_sym(data, key, method, **kwargs): '%s needs an initial value.' % method) _assert_known_encryption_method(method) # AES-256 in CTR mode - if method == EncryptionMethods.AES_256_CTR: + if method == crypto.EncryptionMethods.AES_256_CTR: return AES( key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data) - elif method == EncryptionMethods.XSALSA20: + elif method == crypto.EncryptionMethods.XSALSA20: return XSalsa20( key=key, iv=binascii.a2b_base64(kwargs['iv'])).process(data) @@ -185,11 +178,11 @@ class SoledadCrypto(object): self._soledad = soledad def encrypt_sym(self, data, key, - method=EncryptionMethods.AES_256_CTR): + method=crypto.EncryptionMethods.AES_256_CTR): return encrypt_sym(data, key, method) def decrypt_sym(self, data, key, - method=EncryptionMethods.AES_256_CTR, **kwargs): + method=crypto.EncryptionMethods.AES_256_CTR, **kwargs): return decrypt_sym(data, key, method, **kwargs) def doc_mac_key(self, doc_id, secret): @@ -233,7 +226,8 @@ class SoledadCrypto(object): # Crypto utilities for a SoledadDocument. # -def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret): +def mac_doc(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, enc_iv, + mac_method, secret): """ Calculate a MAC for C{doc} using C{ciphertext}. @@ -249,6 +243,12 @@ def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret): :type doc_rev: str :param ciphertext: The content of the document. :type ciphertext: str + :param enc_scheme: The encryption scheme. + :type enc_scheme: str + :param enc_method: The encryption method. + :type enc_method: str + :param enc_iv: The encryption initialization vector. + :type enc_iv: str :param mac_method: The MAC method to use. :type mac_method: str :param secret: The Soledad storage secret @@ -257,15 +257,20 @@ def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret): :return: The calculated MAC. :rtype: str - :raise UnknownMacMethod: Raised when C{mac_method} is unknown. + :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown. """ try: - soledad_assert(mac_method == MacMethods.HMAC) + soledad_assert(mac_method == crypto.MacMethods.HMAC) except AssertionError: - raise UnknownMacMethod - content = str(doc_id) \ - + str(doc_rev) \ - + ciphertext + raise crypto.UnknownMacMethodError + template = "{doc_id}{doc_rev}{ciphertext}{enc_scheme}{enc_method}{enc_iv}" + content = template.format( + doc_id=doc_id, + doc_rev=doc_rev, + ciphertext=ciphertext, + enc_scheme=enc_scheme, + enc_method=enc_method, + enc_iv=enc_iv) return hmac.new( doc_mac_key(doc_id, secret), content, @@ -297,12 +302,12 @@ def encrypt_docstr(docstr, doc_id, doc_rev, key, secret): string representing the following: { - ENC_JSON_KEY: '', - ENC_SCHEME_KEY: 'symkey', - ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR, - ENC_IV_KEY: '', + crypto.ENC_JSON_KEY: '', + crypto.ENC_SCHEME_KEY: 'symkey', + crypto.ENC_METHOD_KEY: crypto.EncryptionMethods.AES_256_CTR, + crypto.ENC_IV_KEY: '', MAC_KEY: '' - MAC_METHOD_KEY: 'hmac' + crypto.MAC_METHOD_KEY: 'hmac' } :param docstr: A representation of the document to be encrypted. @@ -324,10 +329,10 @@ def encrypt_docstr(docstr, doc_id, doc_rev, key, secret): content. :rtype: str """ - enc_scheme = EncryptionSchemes.SYMKEY - enc_method = EncryptionMethods.AES_256_CTR - mac_method = MacMethods.HMAC - iv, ciphertext = encrypt_sym( + enc_scheme = crypto.EncryptionSchemes.SYMKEY + enc_method = crypto.EncryptionMethods.AES_256_CTR + mac_method = crypto.MacMethods.HMAC + enc_iv, ciphertext = encrypt_sym( str(docstr), # encryption/decryption routines expect str key, method=enc_method) mac = binascii.b2a_hex( # store the mac as hex. @@ -335,6 +340,9 @@ def encrypt_docstr(docstr, doc_id, doc_rev, key, secret): doc_id, doc_rev, ciphertext, + enc_scheme, + enc_method, + enc_iv, mac_method, secret)) # Return a representation for the encrypted content. In the following, we @@ -342,12 +350,12 @@ def encrypt_docstr(docstr, doc_id, doc_rev, key, secret): # serialization does not complain about what it tries to serialize. hex_ciphertext = binascii.b2a_hex(ciphertext) return json.dumps({ - ENC_JSON_KEY: hex_ciphertext, - ENC_SCHEME_KEY: enc_scheme, - ENC_METHOD_KEY: enc_method, - ENC_IV_KEY: iv, - MAC_KEY: mac, - MAC_METHOD_KEY: mac_method, + crypto.ENC_JSON_KEY: hex_ciphertext, + crypto.ENC_SCHEME_KEY: enc_scheme, + crypto.ENC_METHOD_KEY: enc_method, + crypto.ENC_IV_KEY: enc_iv, + crypto.MAC_KEY: mac, + crypto.MAC_METHOD_KEY: mac_method, }) @@ -369,7 +377,8 @@ def decrypt_doc(crypto, doc): return decrypt_doc_dict(doc.content, doc.doc_id, doc.rev, key, secret) -def _verify_doc_mac(doc_id, doc_rev, ciphertext, mac_method, secret, doc_mac): +def _verify_doc_mac(doc_id, doc_rev, ciphertext, enc_scheme, enc_method, + enc_iv, mac_method, secret, doc_mac): """ Verify that C{doc_mac} is a correct MAC for the given document. @@ -379,6 +388,12 @@ def _verify_doc_mac(doc_id, doc_rev, ciphertext, mac_method, secret, doc_mac): :type doc_rev: str :param ciphertext: The content of the document. :type ciphertext: str + :param enc_scheme: The encryption scheme. + :type enc_scheme: str + :param enc_method: The encryption method. + :type enc_method: str + :param enc_iv: The encryption initialization vector. + :type enc_iv: str :param mac_method: The MAC method to use. :type mac_method: str :param secret: The Soledad storage secret @@ -386,12 +401,16 @@ def _verify_doc_mac(doc_id, doc_rev, ciphertext, mac_method, secret, doc_mac): :param doc_mac: The MAC to be verified against. :type doc_mac: str - :raise UnknownMacMethod: Raised when C{mac_method} is unknown. + :raise crypto.UnknownMacMethodError: Raised when C{mac_method} is unknown. + :raise crypto.WrongMacError: Raised when MAC could not be verified. """ calculated_mac = mac_doc( doc_id, doc_rev, ciphertext, + enc_scheme, + enc_method, + enc_iv, mac_method, secret) # we compare mac's hashes to avoid possible timing attacks that might @@ -404,7 +423,8 @@ def _verify_doc_mac(doc_id, doc_rev, ciphertext, mac_method, secret, doc_mac): if doc_mac_hash != calculated_mac_hash: logger.warning("Wrong MAC while decrypting doc...") - raise WrongMac('Could not authenticate document\'s contents.') + raise crypto.WrongMacError("Could not authenticate document's " + "contents.") def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): @@ -416,18 +436,18 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): The passed doc_dict argument should have the following structure: { - ENC_JSON_KEY: '', - ENC_SCHEME_KEY: '', - ENC_METHOD_KEY: '', - ENC_IV_KEY: '', # (optional) + crypto.ENC_JSON_KEY: '', + crypto.ENC_SCHEME_KEY: '', + crypto.ENC_METHOD_KEY: '', + crypto.ENC_IV_KEY: '', # (optional) MAC_KEY: '' - MAC_METHOD_KEY: 'hmac' + crypto.MAC_METHOD_KEY: 'hmac' } C{enc_blob} is the encryption of the JSON serialization of the document's content. For now Soledad just deals with documents whose C{enc_scheme} is - EncryptionSchemes.SYMKEY and C{enc_method} is - EncryptionMethods.AES_256_CTR. + crypto.EncryptionSchemes.SYMKEY and C{enc_method} is + crypto.EncryptionMethods.AES_256_CTR. :param doc_dict: The content of the document to be decrypted. :type doc_dict: dict @@ -447,27 +467,32 @@ def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret): :return: The JSON serialization of the decrypted content. :rtype: str - :raise UnknownEncryptionMethod: Raised when trying to decrypt from an + :raise UnknownEncryptionMethodError: Raised when trying to decrypt from an unknown encryption method. """ # assert document dictionary structure - soledad_assert(ENC_JSON_KEY in doc_dict) - soledad_assert(ENC_SCHEME_KEY in doc_dict) - soledad_assert(ENC_METHOD_KEY in doc_dict) - soledad_assert(ENC_IV_KEY in doc_dict) - soledad_assert(MAC_KEY in doc_dict) - soledad_assert(MAC_METHOD_KEY in doc_dict) - - ciphertext = binascii.a2b_hex(doc_dict[ENC_JSON_KEY]) - enc_scheme = doc_dict[ENC_SCHEME_KEY] - enc_method = doc_dict[ENC_METHOD_KEY] - enc_iv = doc_dict[ENC_IV_KEY] - doc_mac = doc_dict[MAC_KEY] - mac_method = doc_dict[MAC_METHOD_KEY] - - soledad_assert(enc_scheme == EncryptionSchemes.SYMKEY) - - _verify_doc_mac(doc_id, doc_rev, ciphertext, mac_method, secret, doc_mac) + expected_keys = set([ + crypto.ENC_JSON_KEY, + crypto.ENC_SCHEME_KEY, + crypto.ENC_METHOD_KEY, + crypto.ENC_IV_KEY, + crypto.MAC_KEY, + crypto.MAC_METHOD_KEY, + ]) + soledad_assert(expected_keys.issubset(set(doc_dict.keys()))) + + ciphertext = binascii.a2b_hex(doc_dict[crypto.ENC_JSON_KEY]) + enc_scheme = doc_dict[crypto.ENC_SCHEME_KEY] + enc_method = doc_dict[crypto.ENC_METHOD_KEY] + enc_iv = doc_dict[crypto.ENC_IV_KEY] + doc_mac = doc_dict[crypto.MAC_KEY] + mac_method = doc_dict[crypto.MAC_METHOD_KEY] + + soledad_assert(enc_scheme == crypto.EncryptionSchemes.SYMKEY) + + _verify_doc_mac( + doc_id, doc_rev, ciphertext, enc_scheme, enc_method, + enc_iv, mac_method, secret, doc_mac) return decrypt_sym(ciphertext, key, method=enc_method, iv=enc_iv) @@ -481,8 +506,9 @@ def is_symmetrically_encrypted(doc): :rtype: bool """ - if doc.content and ENC_SCHEME_KEY in doc.content: - if doc.content[ENC_SCHEME_KEY] == EncryptionSchemes.SYMKEY: + if doc.content and crypto.ENC_SCHEME_KEY in doc.content: + if doc.content[crypto.ENC_SCHEME_KEY] \ + == crypto.EncryptionSchemes.SYMKEY: return True return False diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index b1c22371..970ac82f 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -33,33 +33,12 @@ from hashlib import sha256 import simplejson as json -from leap.soledad.common import ( - soledad_assert, - soledad_assert_type -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.crypto import ( - MacMethods, - UnknownMacMethod, - WrongMac, - MAC_KEY, - MAC_METHOD_KEY, -) -from leap.soledad.common.errors import ( - InvalidTokenError, - NotLockedError, - AlreadyLockedError, - LockTimedOutError, -) -from leap.soledad.client.events import ( - SOLEDAD_CREATING_KEYS, - SOLEDAD_DONE_CREATING_KEYS, - SOLEDAD_DOWNLOADING_KEYS, - SOLEDAD_DONE_DOWNLOADING_KEYS, - SOLEDAD_UPLOADING_KEYS, - SOLEDAD_DONE_UPLOADING_KEYS, - signal, -) +from leap.soledad.common import soledad_assert +from leap.soledad.common import soledad_assert_type +from leap.soledad.common import document +from leap.soledad.common import errors +from leap.soledad.common import crypto +from leap.soledad.client import events logger = logging.getLogger(name=__name__) @@ -227,9 +206,9 @@ class SoledadSecrets(object): token = timeout = None try: token, timeout = self._shared_db.lock() - except AlreadyLockedError: + except errors.AlreadyLockedError: raise BootstrapSequenceError('Database is already locked.') - except LockTimedOutError: + except errors.LockTimedOutError: raise BootstrapSequenceError('Lock operation timed out.') self._get_or_gen_crypto_secrets() @@ -238,12 +217,12 @@ class SoledadSecrets(object): try: self._shared_db.unlock(token) self._shared_db.close() - except NotLockedError: + except errors.NotLockedError: # for some reason the lock expired. Despite that, secret # loading or generation/storage must have been executed # successfully, so we pass. pass - except InvalidTokenError: + except errors.InvalidTokenError: # here, our lock has not only expired but also some other # client application has obtained a new lock and is currently # doing its thing in the shared database. Using the same @@ -403,8 +382,8 @@ class SoledadSecrets(object): self.KDF_KEY: self.KDF_SCRYPT, self.KDF_SALT_KEY: binascii.b2a_base64(salt), self.KDF_LENGTH_KEY: len(key), - MAC_METHOD_KEY: MacMethods.HMAC, - MAC_KEY: hmac.new( + crypto.MAC_METHOD_KEY: crypto.MacMethods.HMAC, + crypto.MAC_KEY: hmac.new( key, json.dumps(encrypted_secrets), sha256).hexdigest(), @@ -429,13 +408,13 @@ class SoledadSecrets(object): soledad_assert(self.STORAGE_SECRETS_KEY in data) # check mac of the recovery document mac = None - if MAC_KEY in data: - soledad_assert(data[MAC_KEY] is not None) - soledad_assert(MAC_METHOD_KEY in data) + if crypto.MAC_KEY in data: + soledad_assert(data[crypto.MAC_KEY] is not None) + soledad_assert(crypto.MAC_METHOD_KEY in data) soledad_assert(self.KDF_KEY in data) soledad_assert(self.KDF_SALT_KEY in data) soledad_assert(self.KDF_LENGTH_KEY in data) - if data[MAC_METHOD_KEY] == MacMethods.HMAC: + if data[crypto.MAC_METHOD_KEY] == crypto.MacMethods.HMAC: key = scrypt.hash( self._passphrase_as_string(), binascii.a2b_base64(data[self.KDF_SALT_KEY]), @@ -445,10 +424,10 @@ class SoledadSecrets(object): json.dumps(data[self.STORAGE_SECRETS_KEY]), sha256).hexdigest() else: - raise UnknownMacMethod('Unknown MAC method: %s.' % - data[MAC_METHOD_KEY]) - if mac != data[MAC_KEY]: - raise WrongMac('Could not authenticate recovery document\'s ' + raise crypto.UnknownMacMethodError('Unknown MAC method: %s.' % + data[crypto.MAC_METHOD_KEY]) + if mac != data[crypto.MAC_KEY]: + raise crypto.WrongMacError('Could not authenticate recovery document\'s ' 'contents.') # include secrets in the secret pool. secret_count = 0 @@ -469,15 +448,15 @@ class SoledadSecrets(object): database. :return: a document with encrypted key material in its contents - :rtype: SoledadDocument + :rtype: document.SoledadDocument """ - signal(SOLEDAD_DOWNLOADING_KEYS, self._uuid) + events.signal(events.SOLEDAD_DOWNLOADING_KEYS, self._uuid) db = self._shared_db if not db: logger.warning('No shared db found') return doc = db.get_doc(self._shared_db_doc_id()) - signal(SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid) + events.signal(events.SOLEDAD_DONE_DOWNLOADING_KEYS, self._uuid) return doc def _put_secrets_in_shared_db(self): @@ -495,18 +474,18 @@ class SoledadSecrets(object): # try to get secrets doc from server, otherwise create it doc = self._get_secrets_from_shared_db() if doc is None: - doc = SoledadDocument( + doc = document.SoledadDocument( doc_id=self._shared_db_doc_id()) # fill doc with encrypted secrets doc.content = self._export_recovery_document() # upload secrets to server - signal(SOLEDAD_UPLOADING_KEYS, self._uuid) + events.signal(events.SOLEDAD_UPLOADING_KEYS, self._uuid) db = self._shared_db if not db: logger.warning('No shared db found') return db.put_doc(doc) - signal(SOLEDAD_DONE_UPLOADING_KEYS, self._uuid) + events.signal(events.SOLEDAD_DONE_UPLOADING_KEYS, self._uuid) # # Management of secret for symmetric encryption. @@ -618,7 +597,7 @@ class SoledadSecrets(object): Generate a secret for symmetric encryption and store in a local encrypted file. - This method emits the following signals: + This method emits the following events.signals: * SOLEDAD_CREATING_KEYS * SOLEDAD_DONE_CREATING_KEYS @@ -626,13 +605,13 @@ class SoledadSecrets(object): :return: The id of the generated secret. :rtype: str """ - signal(SOLEDAD_CREATING_KEYS, self._uuid) + events.signal(events.SOLEDAD_CREATING_KEYS, self._uuid) # generate random secret secret = os.urandom(self.GEN_SECRET_LENGTH) secret_id = sha256(secret).hexdigest() self._secrets[secret_id] = secret self._store_secrets() - signal(SOLEDAD_DONE_CREATING_KEYS, self._uuid) + events.signal(events.SOLEDAD_DONE_CREATING_KEYS, self._uuid) return secret_id def _store_secrets(self): diff --git a/common/src/leap/soledad/common/crypto.py b/common/src/leap/soledad/common/crypto.py index ab05999b..b4f3234f 100644 --- a/common/src/leap/soledad/common/crypto.py +++ b/common/src/leap/soledad/common/crypto.py @@ -35,7 +35,7 @@ class EncryptionSchemes(object): PUBKEY = 'pubkey' -class UnknownEncryptionScheme(Exception): +class UnknownEncryptionSchemeError(Exception): """ Raised when trying to decrypt from unknown encryption schemes. """ @@ -51,7 +51,7 @@ class EncryptionMethods(object): XSALSA20 = 'xsalsa20' -class UnknownEncryptionMethod(Exception): +class UnknownEncryptionMethodError(Exception): """ Raised when trying to encrypt/decrypt with unknown method. """ @@ -66,7 +66,7 @@ class MacMethods(object): HMAC = 'hmac' -class UnknownMacMethod(Exception): +class UnknownMacMethodError(Exception): """ Raised when trying to authenticate document's content with unknown MAC mehtod. @@ -74,7 +74,7 @@ class UnknownMacMethod(Exception): pass -class WrongMac(Exception): +class WrongMacError(Exception): """ Raised when failing to authenticate document's contents based on MAC. """ diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py index 3081683b..0ab159fd 100644 --- a/common/src/leap/soledad/common/tests/__init__.py +++ b/common/src/leap/soledad/common/tests/__init__.py @@ -27,9 +27,9 @@ from mock import Mock from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.crypto import ENC_SCHEME_KEY from leap.soledad.client import Soledad from leap.soledad.client.crypto import decrypt_doc_dict -from leap.soledad.client.crypto import ENC_SCHEME_KEY from leap.common.testing.basetest import BaseLeapTest diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index 0302a268..f5fb4b7a 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -24,7 +24,13 @@ import binascii from leap.soledad.client import crypto from leap.soledad.common.document import SoledadDocument from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.crypto import WrongMac, UnknownMacMethod +from leap.soledad.common.crypto import WrongMacError +from leap.soledad.common.crypto import UnknownMacMethodError +from leap.soledad.common.crypto import EncryptionMethods +from leap.soledad.common.crypto import ENC_JSON_KEY +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.common.crypto import MAC_KEY +from leap.soledad.common.crypto import MAC_METHOD_KEY class EncryptedSyncTestCase(BaseSoledadTest): @@ -46,8 +52,8 @@ class EncryptedSyncTestCase(BaseSoledadTest): self.assertNotEqual( simpledoc, doc1.content, 'incorrect document encryption') - self.assertTrue(crypto.ENC_JSON_KEY in doc1.content) - self.assertTrue(crypto.ENC_SCHEME_KEY in doc1.content) + self.assertTrue(ENC_JSON_KEY in doc1.content) + self.assertTrue(ENC_SCHEME_KEY in doc1.content) # decrypt doc doc1.set_json(crypto.decrypt_doc(self._soledad._crypto, doc1)) self.assertEqual( @@ -149,13 +155,13 @@ class MacAuthTestCase(BaseSoledadTest): doc.content = simpledoc # encrypt doc doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc)) - self.assertTrue(crypto.MAC_KEY in doc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in doc.content) + self.assertTrue(MAC_KEY in doc.content) + self.assertTrue(MAC_METHOD_KEY in doc.content) # mess with MAC - doc.content[crypto.MAC_KEY] = '1234567890ABCDEF' + doc.content[MAC_KEY] = '1234567890ABCDEF' # try to decrypt doc self.assertRaises( - WrongMac, + WrongMacError, crypto.decrypt_doc, self._soledad._crypto, doc) def test_decrypt_with_unknown_mac_method_raises(self): @@ -167,13 +173,13 @@ class MacAuthTestCase(BaseSoledadTest): doc.content = simpledoc # encrypt doc doc.set_json(crypto.encrypt_doc(self._soledad._crypto, doc)) - self.assertTrue(crypto.MAC_KEY in doc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in doc.content) + self.assertTrue(MAC_KEY in doc.content) + self.assertTrue(MAC_METHOD_KEY in doc.content) # mess with MAC method - doc.content[crypto.MAC_METHOD_KEY] = 'mymac' + doc.content[MAC_METHOD_KEY] = 'mymac' # try to decrypt doc self.assertRaises( - UnknownMacMethod, + UnknownMacMethodError, crypto.decrypt_doc, self._soledad._crypto, doc) @@ -184,20 +190,20 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') plaintext = self._soledad._crypto.decrypt_sym( cyphertext, key, iv=iv, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertEqual('data', plaintext) def test_decrypt_with_wrong_iv_fails(self): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -208,14 +214,14 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): wrongiv = os.urandom(1) + rawiv[1:] plaintext = self._soledad._crypto.decrypt_sym( cyphertext, key, iv=binascii.b2a_base64(wrongiv), - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertNotEqual('data', plaintext) def test_decrypt_with_wrong_key_fails(self): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -225,7 +231,7 @@ class SoledadCryptoAESTestCase(BaseSoledadTest): wrongkey = os.urandom(32) plaintext = self._soledad._crypto.decrypt_sym( cyphertext, wrongkey, iv=iv, - method=crypto.EncryptionMethods.AES_256_CTR) + method=EncryptionMethods.AES_256_CTR) self.assertNotEqual('data', plaintext) @@ -236,20 +242,20 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') plaintext = self._soledad._crypto.decrypt_sym( cyphertext, key, iv=iv, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertEqual('data', plaintext) def test_decrypt_with_wrong_iv_fails(self): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -260,14 +266,14 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): wrongiv = os.urandom(1) + rawiv[1:] plaintext = self._soledad._crypto.decrypt_sym( cyphertext, key, iv=binascii.b2a_base64(wrongiv), - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertNotEqual('data', plaintext) def test_decrypt_with_wrong_key_fails(self): key = os.urandom(32) iv, cyphertext = self._soledad._crypto.encrypt_sym( 'data', key, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertTrue(cyphertext is not None) self.assertTrue(cyphertext != '') self.assertTrue(cyphertext != 'data') @@ -277,5 +283,5 @@ class SoledadCryptoXSalsa20TestCase(BaseSoledadTest): wrongkey = os.urandom(32) plaintext = self._soledad._crypto.decrypt_sym( cyphertext, wrongkey, iv=iv, - method=crypto.EncryptionMethods.XSALSA20) + method=EncryptionMethods.XSALSA20) self.assertNotEqual('data', plaintext) diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 12bfbc3e..31c02fc4 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -28,7 +28,7 @@ from leap.soledad.common.tests import ( ) from leap import soledad from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.crypto import WrongMac +from leap.soledad.common.crypto import WrongMacError from leap.soledad.client import Soledad from leap.soledad.client.sqlcipher import SQLCipherDatabase from leap.soledad.client.secrets import PassphraseTooShort @@ -117,7 +117,7 @@ class AuxMethodsTestCase(BaseSoledadTest): sol.close() self.assertRaises( - WrongMac, + WrongMacError, self._soledad_instance, 'leap@leap.se', passphrase=u'123', prefix=self.rand_prefix) @@ -208,7 +208,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): def setUp(self): # mock signaling soledad.client.signal = Mock() - soledad.client.secrets.signal = Mock() + soledad.client.secrets.events.signal = Mock() # run parent's setUp BaseSoledadTest.setUp(self) @@ -230,57 +230,57 @@ class SoledadSignalingTestCase(BaseSoledadTest): - downloading keys / done downloading keys. - uploading keys / done uploading keys. """ - soledad.client.secrets.signal.reset_mock() + soledad.client.secrets.events.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( secrets_path='alternative_stage3.json', local_db_path='alternative_stage3.u1db') # reverse call order so we can verify in the order the signals were # expected - soledad.client.secrets.signal.mock_calls.reverse() - soledad.client.secrets.signal.call_args = \ - soledad.client.secrets.signal.call_args_list[0] - soledad.client.secrets.signal.call_args_list.reverse() + soledad.client.secrets.events.signal.mock_calls.reverse() + soledad.client.secrets.events.signal.call_args = \ + soledad.client.secrets.events.signal.call_args_list[0] + soledad.client.secrets.events.signal.call_args_list.reverse() # downloading keys signals - soledad.client.secrets.signal.assert_called_with( + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.secrets.signal) - soledad.client.secrets.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.events.signal) + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) # creating keys signals - self._pop_mock_call(soledad.client.secrets.signal) - soledad.client.secrets.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.events.signal) + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_CREATING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.secrets.signal) - soledad.client.secrets.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.events.signal) + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_DONE_CREATING_KEYS, ADDRESS, ) # downloading once more (inside _put_keys_in_shared_db) - self._pop_mock_call(soledad.client.secrets.signal) - soledad.client.secrets.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.events.signal) + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.secrets.signal) - soledad.client.secrets.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.events.signal) + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) # uploading keys signals - self._pop_mock_call(soledad.client.secrets.signal) - soledad.client.secrets.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.events.signal) + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_UPLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.secrets.signal) - soledad.client.secrets.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.events.signal) + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_DONE_UPLOADING_KEYS, ADDRESS, ) @@ -312,7 +312,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): sol.close() # reset mock - soledad.client.secrets.signal.reset_mock() + soledad.client.secrets.events.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( secrets_path='alternative_stage2.json', @@ -320,17 +320,17 @@ class SoledadSignalingTestCase(BaseSoledadTest): shared_db_class=Stage2MockSharedDB) # reverse call order so we can verify in the order the signals were # expected - soledad.client.secrets.signal.mock_calls.reverse() - soledad.client.secrets.signal.call_args = \ - soledad.client.secrets.signal.call_args_list[0] - soledad.client.secrets.signal.call_args_list.reverse() + soledad.client.secrets.events.signal.mock_calls.reverse() + soledad.client.secrets.events.signal.call_args = \ + soledad.client.secrets.events.signal.call_args_list[0] + soledad.client.secrets.events.signal.call_args_list.reverse() # assert download keys signals - soledad.client.secrets.signal.assert_called_with( + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_DOWNLOADING_KEYS, ADDRESS, ) - self._pop_mock_call(soledad.client.secrets.signal) - soledad.client.secrets.signal.assert_called_with( + self._pop_mock_call(soledad.client.secrets.events.signal) + soledad.client.secrets.events.signal.assert_called_with( proto.SOLEDAD_DONE_DOWNLOADING_KEYS, ADDRESS, ) -- cgit v1.2.3 From 371651d5d5ca378be92d6d3a0dcfc0f8467b78b7 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 5 Dec 2014 10:40:10 -0200 Subject: Update debian package building script. --- scripts/build_debian_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_debian_package.sh b/scripts/build_debian_package.sh index 1ec9b00a..b9fb93a9 100755 --- a/scripts/build_debian_package.sh +++ b/scripts/build_debian_package.sh @@ -26,7 +26,7 @@ export GIT_DIR=${workdir}/soledad/.git export GIT_WORK_TREE=${workdir}/soledad git remote add leapcode ${SOLEDAD_MAIN_REPO} git fetch leapcode -git checkout -b debian leapcode/debian +git checkout -b debian/experimental leapcode/debian/experimental git merge --no-edit ${branch} (cd ${workdir}/soledad && debuild -uc -us) echo "Packages generated in ${workdir}" -- cgit v1.2.3 From eda955ed1f761d8de005a2f2c03fc7d10484ac28 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 5 Dec 2014 14:35:06 -0200 Subject: Add key manager to client db access script. --- scripts/db_access/client_side_db.py | 86 ++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 67c5dbe1..2b1b7c72 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -2,23 +2,17 @@ # This script gives client-side access to one Soledad user database. - -import sys import os import argparse -import re import tempfile import getpass import requests -import json import srp._pysrp as srp import binascii import logging - -from leap.common.config import get_path_prefix from leap.soledad.client import Soledad - +from leap.keymanager import KeyManager from util import ValidateUserHandle @@ -33,30 +27,30 @@ safe_unhexlify = lambda x: binascii.unhexlify(x) if ( len(x) % 2 == 0) else binascii.unhexlify('0' + x) -def fail(reason): +def _fail(reason): logger.error('Fail: ' + reason) exit(2) -def get_api_info(provider): +def _get_api_info(provider): info = requests.get( 'https://'+provider+'/provider.json', verify=False).json() return info['api_uri'], info['api_version'] -def login(username, passphrase, provider, api_uri, api_version): +def _login(username, passphrase, provider, api_uri, api_version): usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) auth = None try: - auth = authenticate(api_uri, api_version, usr).json() + auth = _authenticate(api_uri, api_version, usr).json() except requests.exceptions.ConnectionError: - fail('Could not connect to server.') + _fail('Could not connect to server.') if 'errors' in auth: - fail(str(auth['errors'])) + _fail(str(auth['errors'])) return api_uri, api_version, auth -def authenticate(api_uri, api_version, usr): +def _authenticate(api_uri, api_version, usr): api_url = "%s/%s" % (api_uri, api_version) session = requests.session() uname, A = usr.start_authentication() @@ -64,16 +58,16 @@ def authenticate(api_uri, api_version, usr): init = session.post( api_url + '/sessions', data=params, verify=False).json() if 'errors' in init: - fail('test user not found') + _fail('test user not found') M = usr.process_challenge( safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) return session.put(api_url + '/sessions/' + uname, verify=False, data={'client_auth': binascii.hexlify(M)}) -def get_soledad_info(username, provider, passphrase, basedir): - api_uri, api_version = get_api_info(provider) - auth = login(username, passphrase, provider, api_uri, api_version) +def _get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = _get_api_info(provider) + auth = _login(username, passphrase, provider, api_uri, api_version) # get soledad server url service_url = '%s/%s/config/soledad-service.json' % \ (api_uri, api_version) @@ -101,10 +95,9 @@ def get_soledad_info(username, provider, passphrase, basedir): return auth[2]['id'], server_url, cert_file, auth[2]['token'] -def get_soledad_instance(username, provider, passphrase, basedir): +def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, + token): # setup soledad info - uuid, server_url, cert_file, token = \ - get_soledad_info(username, provider, passphrase, basedir) logger.info('UUID is %s' % uuid) logger.info('Server URL is %s' % server_url) secrets_path = os.path.join( @@ -123,10 +116,22 @@ def get_soledad_instance(username, provider, passphrase, basedir): defer_encryption=False) -# main program +def _get_keymanager_instance(username, provider, soledad, token, + ca_cert_path=None, api_uri=None, api_version=None, uid=None, + gpgbinary=None): + return KeyManager( + "{username}@{provider}".format(username=username, provider=provider), + "http://uri", + soledad, + token=token, + ca_cert_path=ca_cert_path, + api_uri=api_uri, + api_version=api_version, + uid=uid, + gpgbinary=gpgbinary) -if __name__ == '__main__': +def _parse_args(): # parse command line parser = argparse.ArgumentParser() parser.add_argument( @@ -137,21 +142,42 @@ if __name__ == '__main__': parser.add_argument( '-p', dest='passphrase', required=False, default=None, help='the user passphrase') - args = parser.parse_args() + return parser.parse_args() - # get the password + +def _get_passphrase(args): passphrase = args.passphrase if passphrase is None: passphrase = getpass.getpass( 'Password for %s@%s: ' % (args.username, args.provider)) + return passphrase + - # get the basedir +def _get_basedir(args): basedir = args.basedir if basedir is None: basedir = tempfile.mkdtemp() logger.info('Using %s as base directory.' % basedir) + return basedir + + +# main program + +if __name__ == '__main__': + args = _parse_args() + passphrase = _get_passphrase(args) + basedir = _get_basedir(args) + uuid, server_url, cert_file, token = \ + _get_soledad_info(args.username, args.provider, passphrase, basedir) + + soledad = _get_soledad_instance( + uuid, passphrase, basedir, server_url, cert_file, token) + soledad.sync() + + km = _get_keymanager_instance( + args.username, + args.provider, + soledad, + token, + uid=uuid) - # get the soledad instance - s = get_soledad_instance( - args.username, args.provider, passphrase, basedir) - s.sync() -- cgit v1.2.3 From 5f2781c807ba5c645895cf6c42321eb5bcfeef15 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 8 Dec 2014 14:49:33 -0200 Subject: Add compatibility note to README.rst file. --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 93922413..5d6593ca 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,13 @@ repository: :target: https://crate.io/packages/leap.soledad.server +Compatibility +------------- + +* Server 0.7.x is incompatible with client < 0.7.0 because of modifications on + encrypted document MAC calculation. + + Tests ----- -- cgit v1.2.3 From 69d41302f6058f79ef565f5b3f4d88d38974a028 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 8 Dec 2014 11:40:33 -0600 Subject: update debian branch in repackaging howto --- docs/debian-repackaging.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/debian-repackaging.rst b/docs/debian-repackaging.rst index a7488a84..888d6c03 100644 --- a/docs/debian-repackaging.rst +++ b/docs/debian-repackaging.rst @@ -6,7 +6,7 @@ How to repackage latest code Enter debian branch:: - git checkout debian + git checkout debian/experimental Merge your latest and greatest:: -- cgit v1.2.3 From d234ec94734219116b1190232b6ba9c1a118e1d6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 9 Dec 2014 16:07:17 -0600 Subject: Fix incorrect ssl context setup The changes introduced in aafa79c0f5 having to do with the cert verification are incorrect, regarding the use of the newest ssl context api introduced in python 2.7.9. There the use of the server setup was taken, instead of the correct client options. I hereby apologize for the insuficient testing on that fix. It happens that I wrongly tested in an evironment that did the fallback to pre-2.7.9 interpreter. --- client/src/leap/soledad/client/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index d7d01b57..0750dfbe 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -819,10 +819,9 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection): ctx.options |= ssl.OP_NO_SSLv2 ctx.options |= ssl.OP_NO_SSLv3 - ctx.load_cert_chain(certfile=SOLEDAD_CERT) + ctx.load_verify_locations(cafile=SOLEDAD_CERT) ctx.verify_mode = ssl.CERT_REQUIRED - self.sock = ctx.wrap_socket( - sock, server_side=True, server_hostname=self.host) + self.sock = ctx.wrap_socket(sock) except AttributeError: self.sock = ssl.wrap_socket( -- cgit v1.2.3 From e909a218efb0ad31f413c47c90303f44f6906158 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 16 Dec 2014 14:47:21 -0200 Subject: Fix server initscript location (#6557). --- .../bug_6557_fix-server-initscript-location | 1 + server/pkg/soledad | 73 ---------------------- server/pkg/soledad-server | 73 ++++++++++++++++++++++ server/setup.py | 2 +- 4 files changed, 75 insertions(+), 74 deletions(-) create mode 100644 server/changes/bug_6557_fix-server-initscript-location delete mode 100644 server/pkg/soledad create mode 100644 server/pkg/soledad-server diff --git a/server/changes/bug_6557_fix-server-initscript-location b/server/changes/bug_6557_fix-server-initscript-location new file mode 100644 index 00000000..6032b302 --- /dev/null +++ b/server/changes/bug_6557_fix-server-initscript-location @@ -0,0 +1 @@ + o Fix server initscript location (#6557). diff --git a/server/pkg/soledad b/server/pkg/soledad deleted file mode 100644 index ccb3e9b0..00000000 --- a/server/pkg/soledad +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/sh -### BEGIN INIT INFO -# Provides: soledad -# Required-Start: $network $named $remote_fs $syslog $time -# Required-Stop: $network $named $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start soledad daemon at boot time -# Description: Synchronization of locally encrypted data among devices -### END INIT INFO - -PATH=/sbin:/bin:/usr/sbin:/usr/bin -PIDFILE=/var/run/soledad.pid -RUNDIR=/var/lib/soledad/ -OBJ=leap.soledad.server.application -LOGFILE=/var/log/soledad.log -HTTPS_PORT=2424 -CERT_PATH=/etc/leap/soledad-server.pem -PRIVKEY_PATH=/etc/leap/soledad-server.key -TWISTD_PATH=/usr/bin/twistd -HOME=/var/lib/soledad/ -SSL_METHOD=SSLv23_METHOD -USER=soledad -GROUP=soledad - -[ -r /etc/default/soledad ] && . /etc/default/soledad - -test -r /etc/leap/ || exit 0 - -. /lib/lsb/init-functions - - -case "${1}" in - start) - echo -n "Starting soledad: twistd" - start-stop-daemon --start --quiet \ - --user=${USER} --group=${GROUP} \ - --exec ${TWISTD_PATH} -- \ - --pidfile=${PIDFILE} \ - --logfile=${LOGFILE} \ - web \ - --wsgi=${OBJ} \ - --port=ssl:${HTTPS_PORT}:privateKey=${PRIVKEY_PATH}:certKey=${CERT_PATH}:sslmethod=${SSL_METHOD} - echo "." - ;; - - stop) - echo -n "Stopping soledad: twistd" - start-stop-daemon --stop --quiet \ - --pidfile ${PIDFILE} - echo "." - ;; - - restart) - ${0} stop - ${0} start - ;; - - force-reload) - ${0} restart - ;; - - status) - status_of_proc -p ${PIDFILE} ${TWISTD_PATH} soledad && exit 0 || exit ${?} - ;; - - *) - echo "Usage: /etc/init.d/soledad {start|stop|restart|force-reload|status}" >&2 - exit 1 - ;; -esac - -exit 0 diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server new file mode 100644 index 00000000..ccb3e9b0 --- /dev/null +++ b/server/pkg/soledad-server @@ -0,0 +1,73 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: soledad +# Required-Start: $network $named $remote_fs $syslog $time +# Required-Stop: $network $named $remote_fs $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Start soledad daemon at boot time +# Description: Synchronization of locally encrypted data among devices +### END INIT INFO + +PATH=/sbin:/bin:/usr/sbin:/usr/bin +PIDFILE=/var/run/soledad.pid +RUNDIR=/var/lib/soledad/ +OBJ=leap.soledad.server.application +LOGFILE=/var/log/soledad.log +HTTPS_PORT=2424 +CERT_PATH=/etc/leap/soledad-server.pem +PRIVKEY_PATH=/etc/leap/soledad-server.key +TWISTD_PATH=/usr/bin/twistd +HOME=/var/lib/soledad/ +SSL_METHOD=SSLv23_METHOD +USER=soledad +GROUP=soledad + +[ -r /etc/default/soledad ] && . /etc/default/soledad + +test -r /etc/leap/ || exit 0 + +. /lib/lsb/init-functions + + +case "${1}" in + start) + echo -n "Starting soledad: twistd" + start-stop-daemon --start --quiet \ + --user=${USER} --group=${GROUP} \ + --exec ${TWISTD_PATH} -- \ + --pidfile=${PIDFILE} \ + --logfile=${LOGFILE} \ + web \ + --wsgi=${OBJ} \ + --port=ssl:${HTTPS_PORT}:privateKey=${PRIVKEY_PATH}:certKey=${CERT_PATH}:sslmethod=${SSL_METHOD} + echo "." + ;; + + stop) + echo -n "Stopping soledad: twistd" + start-stop-daemon --stop --quiet \ + --pidfile ${PIDFILE} + echo "." + ;; + + restart) + ${0} stop + ${0} start + ;; + + force-reload) + ${0} restart + ;; + + status) + status_of_proc -p ${PIDFILE} ${TWISTD_PATH} soledad && exit 0 || exit ${?} + ;; + + *) + echo "Usage: /etc/init.d/soledad {start|stop|restart|force-reload|status}" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/server/setup.py b/server/setup.py index 573622ce..124ddd32 100644 --- a/server/setup.py +++ b/server/setup.py @@ -35,7 +35,7 @@ if isset('VIRTUAL_ENV') or isset('LEAP_SKIP_INIT'): data_files = None else: # XXX this should go only for linux/mac - data_files = [("/etc/init.d/", ["pkg/soledad"])] + data_files = [("/etc/init.d/", ["pkg/soledad-server"])] trove_classifiers = ( -- cgit v1.2.3 From fa8dacef003d30cd9b56f7e2b07baa3b387c1e20 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 18 Dec 2014 14:42:13 -0200 Subject: Update testing scripts. --- scripts/db_access/reset_db.py | 132 ++++++++++++++++++------- scripts/ddocs/update_design_docs.py | 191 ++++++++++++++++++++---------------- scripts/profiling/spam.py | 123 +++++++++++++++++++++++ 3 files changed, 325 insertions(+), 121 deletions(-) create mode 100755 scripts/profiling/spam.py diff --git a/scripts/db_access/reset_db.py b/scripts/db_access/reset_db.py index 80871856..7c6d281b 100644 --- a/scripts/db_access/reset_db.py +++ b/scripts/db_access/reset_db.py @@ -5,20 +5,21 @@ # WARNING: running this script over a database will delete all documents but # the one with id u1db_config (which contains db metadata) and design docs # needed for couch backend. +# +# Run it like this to get some help: +# +# ./reset_db.py --help -import sys -from ConfigParser import ConfigParser import threading import logging -from couchdb import Database as CouchDatabase - +import argparse +import re -if len(sys.argv) != 2: - print 'Usage: %s ' % sys.argv[0] - exit(1) -uuid = sys.argv[1] +from ConfigParser import ConfigParser +from couchdb import Database as CouchDatabase +from couchdb import Server as CouchServer # create a logger @@ -27,23 +28,6 @@ LOG_FORMAT = '%(asctime)s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) -# get couch url -cp = ConfigParser() -cp.read('/etc/leap/soledad-server.conf') -url = cp.get('soledad-server', 'couch_url') - - -# confirm -yes = raw_input("Are you sure you want to reset the database for user %s " - "(type YES)? " % uuid) -if yes != 'YES': - print 'Bailing out...' - exit(2) - - -db = CouchDatabase('%s/user-%s' % (url, uuid)) - - class _DeleterThread(threading.Thread): def __init__(self, db, doc_id, release_fun): @@ -59,21 +43,95 @@ class _DeleterThread(threading.Thread): self._release_fun() -semaphore_pool = threading.BoundedSemaphore(value=20) - - -threads = [] -for doc_id in db: - if doc_id != 'u1db_config' and not doc_id.startswith('_design'): +def get_confirmation(noconfirm, uuid, shared): + msg = "Are you sure you want to reset %s (type YES)? " + if shared: + msg = msg % "the shared database" + elif uuid: + msg = msg % ("the database for user %s" % uuid) + else: + msg = msg % "all databases" + if noconfirm is False: + yes = raw_input(msg) + if yes != 'YES': + print 'Bailing out...' + exit(2) + + +def get_url(empty): + url = None + if empty is False: + # get couch url + cp = ConfigParser() + cp.read('/etc/leap/soledad-server.conf') + url = cp.get('soledad-server', 'couch_url') + else: + with open('/etc/couchdb/couchdb.netrc') as f: + netrc = f.read() + admin_password = re.match('^.* password (.*)$', netrc).groups()[0] + url = 'http://admin:%s@127.0.0.1:5984' % admin_password + return url + + +def reset_all_dbs(url, empty): + server = CouchServer('%s' % (url)) + for dbname in server: + if dbname.startswith('user-') or dbname == 'shared': + reset_db(url, dbname, empty) + + +def reset_db(url, dbname, empty): + db = CouchDatabase('%s/%s' % (url, dbname)) + semaphore_pool = threading.BoundedSemaphore(value=20) + + # launch threads for deleting docs + threads = [] + for doc_id in db: + if empty is False: + if doc_id == 'u1db_config' or doc_id.startswith('_design'): + continue semaphore_pool.acquire() logger.info('[main] launching thread for doc: %s' % doc_id) t = _DeleterThread(db, doc_id, semaphore_pool.release) t.start() threads.append(t) - -logger.info('[main] waiting for threads.') -map(lambda thread: thread.join(), threads) - - -logger.info('[main] done.') + # wait for threads to finish + logger.info('[main] waiting for threads.') + map(lambda thread: thread.join(), threads) + logger.info('[main] done.') + + +def _parse_args(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument('-u', dest='uuid', default=False, + help='Reset database of given user.') + group.add_argument('-s', dest='shared', action='store_true', default=False, + help='Reset the shared database.') + group.add_argument('-a', dest='all', action='store_true', default=False, + help='Reset all user databases.') + parser.add_argument( + '-e', dest='empty', action='store_true', required=False, default=False, + help='Empty database (do not preserve minimal set of u1db documents).') + parser.add_argument( + '-y', dest='noconfirm', action='store_true', required=False, + default=False, + help='Do not ask for confirmation.') + return parser.parse_args(), parser + + +if __name__ == '__main__': + args, parser = _parse_args() + if not (args.uuid or args.shared or args.all): + parser.print_help() + exit(1) + + url = get_url(args.empty) + get_confirmation(args.noconfirm, args.uuid, args.shared) + if args.uuid: + reset_db(url, "user-%s" % args.uuid, args.empty) + elif args.shared: + reset_db(url, "shared", args.empty) + elif args.all: + reset_all_dbs(url, args.empty) diff --git a/scripts/ddocs/update_design_docs.py b/scripts/ddocs/update_design_docs.py index e7b5a29c..2e2fa8f0 100644 --- a/scripts/ddocs/update_design_docs.py +++ b/scripts/ddocs/update_design_docs.py @@ -11,84 +11,83 @@ import re import threading import binascii - +from urlparse import urlparse from getpass import getpass from ConfigParser import ConfigParser -from couchdb.client import Server -from couchdb.http import Resource, Session -from datetime import datetime -from urlparse import urlparse +from couchdb.client import Server +from couchdb.http import Resource +from couchdb.http import Session +from couchdb.http import ResourceNotFound from leap.soledad.common import ddocs -# parse command line for the log file name -logger_fname = "/tmp/update-design-docs_%s.log" % \ - str(datetime.now()).replace(' ', '_') -parser = argparse.ArgumentParser() -parser.add_argument('--log', action='store', default=logger_fname, type=str, - required=False, help='the name of the log file', nargs=1) -args = parser.parse_args() +MAX_THREADS = 20 +DESIGN_DOCS = { + '_design/docs': json.loads(binascii.a2b_base64(ddocs.docs)), + '_design/syncs': json.loads(binascii.a2b_base64(ddocs.syncs)), + '_design/transactions': json.loads( + binascii.a2b_base64(ddocs.transactions)), +} -# configure the logger +# create a logger logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -print "Logging to %s." % args.log -logging.basicConfig( - filename=args.log, - format="%(asctime)-15s %(message)s") +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) -# configure threads -max_threads = 20 -semaphore_pool = threading.BoundedSemaphore(value=max_threads) -threads = [] +def _parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-u', dest='uuid', default=None, type=str, + help='the UUID of the user') + parser.add_argument('-t', dest='threads', default=MAX_THREADS, type=int, + help='the number of parallel threads') + return parser.parse_args() -# get couch url -cp = ConfigParser() -cp.read('/etc/leap/soledad-server.conf') -url = urlparse(cp.get('soledad-server', 'couch_url')) -# get admin password -netloc = re.sub('^.*@', '', url.netloc) -url = url._replace(netloc=netloc) -password = getpass("Admin password for %s: " % url.geturl()) -url = url._replace(netloc='admin:%s@%s' % (password, netloc)) +def _get_url(): + # get couch url + cp = ConfigParser() + cp.read('/etc/leap/soledad-server.conf') + url = urlparse(cp.get('soledad-server', 'couch_url')) + # get admin password + netloc = re.sub('^.*@', '', url.netloc) + url = url._replace(netloc=netloc) + password = getpass("Admin password for %s: " % url.geturl()) + return url._replace(netloc='admin:%s@%s' % (password, netloc)) -resource = Resource(url.geturl(), Session(retry_delays=[1,2,4,8], timeout=10)) -server = Server(url=resource) -hidden_url = re.sub( - 'http://(.*):.*@', - 'http://\\1:xxxxx@', - url.geturl()) +def _get_server(url): + resource = Resource( + url.geturl(), Session(retry_delays=[1, 2, 4, 8], timeout=10)) + return Server(url=resource) -print """ -========== -ATTENTION! -========== -This script will modify Soledad's shared and user databases in: +def _confirm(url): + hidden_url = re.sub( + 'http://(.*):.*@', + 'http://\\1:xxxxx@', + url.geturl()) - %s + print """ + ========== + ATTENTION! + ========== -This script does not make a backup of the couch db data, so make sure you -have a copy or you may loose data. -""" % hidden_url -confirm = raw_input("Proceed (type uppercase YES)? ") + This script will modify Soledad's shared and user databases in: -if confirm != "YES": - exit(1) + %s -# convert design doc content + This script does not make a backup of the couch db data, so make sure you + have a copy or you may loose data. + """ % hidden_url + confirm = raw_input("Proceed (type uppercase YES)? ") + + if confirm != "YES": + exit(1) -design_docs = { - '_design/docs': json.loads(binascii.a2b_base64(ddocs.docs)), - '_design/syncs': json.loads(binascii.a2b_base64(ddocs.syncs)), - '_design/transactions': json.loads(binascii.a2b_base64(ddocs.transactions)), -} # # Thread @@ -106,42 +105,66 @@ class DBWorkerThread(threading.Thread): def run(self): - logger.info("(%d/%d) Updating db %s." % (self._db_idx, self._db_len, - self._dbname)) + logger.info( + "(%d/%d) Updating db %s." + % (self._db_idx, self._db_len, self._dbname)) - for doc_id in design_docs: - doc = self._cdb[doc_id] + for doc_id in DESIGN_DOCS: + try: + doc = self._cdb[doc_id] + except ResourceNotFound: + doc = {'_id': doc_id} for key in ['lists', 'views', 'updates']: - if key in design_docs[doc_id]: - doc[key] = design_docs[doc_id][key] + if key in DESIGN_DOCS[doc_id]: + doc[key] = DESIGN_DOCS[doc_id][key] self._cdb.save(doc) # release the semaphore self._release_fun() -db_idx = 0 -db_len = len(server) -for dbname in server: - - db_idx += 1 - - if not (dbname.startswith('user-') or dbname == 'shared') \ - or dbname == 'user-test-db': - logger.info("(%d/%d) Skipping db %s." % (db_idx, db_len, dbname)) - continue - - - # get access to couch db - cdb = Server(url.geturl())[dbname] - - #--------------------------------------------------------------------- - # Start DB worker thread - #--------------------------------------------------------------------- - semaphore_pool.acquire() - thread = DBWorkerThread(server, dbname, db_idx, db_len, semaphore_pool.release) +def _launch_update_design_docs_thread( + server, dbname, db_idx, db_len, semaphore_pool): + semaphore_pool.acquire() # wait for an available working slot + thread = DBWorkerThread( + server, dbname, db_idx, db_len, semaphore_pool.release) thread.daemon = True thread.start() - threads.append(thread) - -map(lambda thread: thread.join(), threads) + return thread + + +def _update_design_docs(args, server): + + # find the actual databases to be updated + dbs = [] + if args.uuid: + dbs.append('user-%s' % args.uuid) + else: + for dbname in server: + if dbname.startswith('user-') or dbname == 'shared': + dbs.append(dbname) + else: + logger.info("Skipping db %s." % dbname) + + db_idx = 0 + db_len = len(dbs) + semaphore_pool = threading.BoundedSemaphore(value=args.threads) + threads = [] + + # launch the update + for db in dbs: + db_idx += 1 + threads.append( + _launch_update_design_docs_thread( + server, db, db_idx, db_len, semaphore_pool)) + + # wait for all threads to finish + map(lambda thread: thread.join(), threads) + + +if __name__ == "__main__": + args = _parse_args() + url = _get_url() + _confirm(url) + server = _get_server(url) + _update_design_docs(args, server) diff --git a/scripts/profiling/spam.py b/scripts/profiling/spam.py new file mode 100755 index 00000000..091a8c48 --- /dev/null +++ b/scripts/profiling/spam.py @@ -0,0 +1,123 @@ +#!/usr/bin/python + +# Send a lot of messages in parallel. + + +import string +import smtplib +import threading +import logging + +from argparse import ArgumentParser + + +SMTP_HOST = 'chipmonk.cdev.bitmask.net' +NUMBER_OF_THREADS = 20 + + +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + +def _send_email(host, subject, to_addr, from_addr, body_text): + """ + Send an email + """ + body = string.join(( + "From: %s" % from_addr, + "To: %s" % to_addr, + "Subject: %s" % subject, + "", + body_text + ), "\r\n") + server = smtplib.SMTP(host) + server.sendmail(from_addr, [to_addr], body) + server.quit() + + +def _parse_args(): + parser = ArgumentParser() + parser.add_argument( + 'target_address', + help='The target email address to spam') + parser.add_argument( + 'number_of_messages', type=int, + help='The amount of messages email address to spam') + parser.add_argument( + '-s', dest='server', default=SMTP_HOST, + help='The SMTP server to use') + parser.add_argument( + '-t', dest='threads', default=NUMBER_OF_THREADS, + help='The maximum number of parallel threads to launch') + return parser.parse_args() + + +class EmailSenderThread(threading.Thread): + + def __init__(self, host, subject, to_addr, from_addr, body_text, + finished_fun): + threading.Thread.__init__(self) + self._host = host + self._subject = subject + self._to_addr = to_addr + self._from_addr = from_addr + self._body_text = body_text + self._finished_fun = finished_fun + + def run(self): + _send_email( + self._host, self._subject, self._to_addr, self._from_addr, + self._body_text) + self._finished_fun() + + +def _launch_email_thread(host, subject, to_addr, from_addr, body_text, + finished_fun): + thread = EmailSenderThread( + host, subject, to_addr, from_addr, body_text, finished_fun) + thread.start() + return thread + + +class FinishedThreads(object): + + def __init__(self): + self._finished = 0 + self._lock = threading.Lock() + + def signal(self): + with self._lock: + self._finished = self._finished + 1 + logger.info('Number of messages sent: %d.' % self._finished) + + +def _send_messages(args): + host = args.server + subject = "Message from Soledad script" + to_addr = args.target_address + from_addr = args.target_address + body_text = "Test message" + + semaphore = threading.Semaphore(args.threads) + threads = [] + finished_threads = FinishedThreads() + + def _finished_fun(): + semaphore.release() + finished_threads.signal() + + for i in xrange(args.number_of_messages): + semaphore.acquire() + threads.append( + _launch_email_thread( + host, subject, to_addr, from_addr, body_text, + _finished_fun)) + + for t in threads: + t.join() + + +if __name__ == "__main__": + args = _parse_args() + _send_messages(args) -- cgit v1.2.3 From 4c918100110029ccbab463bd77b3565383fc409b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 16 Sep 2014 22:09:06 -0500 Subject: remove unused imports --- client/src/leap/soledad/client/shared_db.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index 52e51c6f..31c4e8e8 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -14,19 +14,11 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ A shared database for storing/retrieving encrypted key material. """ - -import simplejson as json - - from u1db.remote import http_database - -from leap.soledad.common import SHARED_DB_LOCK_DOC_ID_PREFIX from leap.soledad.client.auth import TokenBasedAuth -- cgit v1.2.3 From 753784542944d12ed736f4c89bd24cb95cb2afbb Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 8 Oct 2014 02:00:46 +0200 Subject: remove taskthread dependency --- client/pkg/requirements.pip | 1 - 1 file changed, 1 deletion(-) diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index c694182d..61258f01 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -4,7 +4,6 @@ u1db scrypt pycryptopp cchardet -taskthread zope.proxy # -- cgit v1.2.3 From 238822f869f8210883a82f87ae66a48751a7321b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 23 Sep 2014 12:43:14 -0500 Subject: use max cpu_count workers on pool --- client/src/leap/soledad/client/crypto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index d6d9a618..aa8135c0 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -521,7 +521,7 @@ class SyncEncryptDecryptPool(object): """ Base class for encrypter/decrypter pools. """ - WORKERS = 5 + WORKERS = multiprocessing.cpu_count() def __init__(self, crypto, sync_db, write_lock): """ @@ -590,7 +590,7 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): of documents to be synced. """ # TODO implement throttling to reduce cpu usage?? - WORKERS = 5 + WORKERS = multiprocessing.cpu_count() TABLE_NAME = "docs_tosync" FIELD_NAMES = "doc_id, rev, content" -- cgit v1.2.3 From 091c2034ee08956541c1111673ebe2f69673f9f8 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 16 Sep 2014 23:23:20 -0500 Subject: reorganize pragmas, stub SQLCipherOptions object --- client/src/leap/soledad/client/__init__.py | 3 +- client/src/leap/soledad/client/adbapi.py | 77 ++++ client/src/leap/soledad/client/mp_safe_db.py | 2 +- client/src/leap/soledad/client/pragmas.py | 349 +++++++++++++++++ client/src/leap/soledad/client/sqlcipher.py | 565 ++++++--------------------- 5 files changed, 538 insertions(+), 458 deletions(-) create mode 100644 client/src/leap/soledad/client/adbapi.py create mode 100644 client/src/leap/soledad/client/pragmas.py diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 0750dfbe..50fcff2c 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -374,7 +374,8 @@ class Soledad(object): include_deleted=include_deleted) def get_all_docs(self, include_deleted=False): - """Get the JSON content for all documents in the database. + """ + Get the JSON content for all documents in the database. :param include_deleted: If set to True, deleted documents will be returned with empty content. Otherwise deleted diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py new file mode 100644 index 00000000..730999a3 --- /dev/null +++ b/client/src/leap/soledad/client/adbapi.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# sqlcipher.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +An asyncrhonous interface to soledad using sqlcipher backend. +It uses twisted.enterprise.adbapi. + +""" +import os +import sys + +from twisted.enterprise import adbapi +from twisted.python import log + +DEBUG_SQL = os.environ.get("LEAP_DEBUG_SQL") +if DEBUG_SQL: + log.startLogging(sys.stdout) + + +def getConnectionPool(db=None, key=None): + return SQLCipherConnectionPool( + "pysqlcipher.dbapi2", database=db, key=key, check_same_thread=False) + + +class SQLCipherConnectionPool(adbapi.ConnectionPool): + + key = None + + def connect(self): + """ + Return a database connection when one becomes available. + + This method blocks and should be run in a thread from the internal + threadpool. Don't call this method directly from non-threaded code. + Using this method outside the external threadpool may exceed the + maximum number of connections in the pool. + + :return: a database connection from the pool. + """ + self.noisy = DEBUG_SQL + + tid = self.threadID() + conn = self.connections.get(tid) + + if self.key is None: + self.key = self.connkw.pop('key', None) + + if conn is None: + if self.noisy: + log.msg('adbapi connecting: %s %s%s' % (self.dbapiName, + self.connargs or '', + self.connkw or '')) + conn = self.dbapi.connect(*self.connargs, **self.connkw) + + # XXX we should hook here all OUR SOLEDAD pragmas ----- + conn.cursor().execute("PRAGMA key=%s" % self.key) + conn.commit() + # ----------------------------------------------------- + # XXX profit of openfun isntead??? + + if self.openfun is not None: + self.openfun(conn) + self.connections[tid] = conn + return conn diff --git a/client/src/leap/soledad/client/mp_safe_db.py b/client/src/leap/soledad/client/mp_safe_db.py index 780b7153..9ed0bef4 100644 --- a/client/src/leap/soledad/client/mp_safe_db.py +++ b/client/src/leap/soledad/client/mp_safe_db.py @@ -88,7 +88,7 @@ class MPSafeSQLiteDB(Thread): res = Queue() self.execute(req, arg, res) while True: - rec=res.get() + rec = res.get() if rec == self.NO_MORE: break yield rec diff --git a/client/src/leap/soledad/client/pragmas.py b/client/src/leap/soledad/client/pragmas.py new file mode 100644 index 00000000..a21e68a8 --- /dev/null +++ b/client/src/leap/soledad/client/pragmas.py @@ -0,0 +1,349 @@ +# -*- coding: utf-8 -*- +# pragmas.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Different pragmas used in the SQLCIPHER database. +""" +# TODO --------------------------------------------------------------- +# Work In Progress. +# We need to reduce the impedance mismatch between the current soledad +# implementation and the eventually asynchronous api. +# So... how to plug it in, allowing for an optional sync / async coexistence? +# One of the first things is to isolate all the pragmas work that has to be +# done during initialization. +# And, instead of having all of them passed the db_handle and executing that, +# we could have just a string returned, that can be chained to a deferred. +# --------------------------------------------------------------------- +import logging +import string + +logger = logging.getLogger(__name__) + + +def set_crypto_pragmas(db_handle, sqlcipher_opts): + """ + Set cryptographic params (key, cipher, KDF number of iterations and + cipher page size). + + :param db_handle: + :type db_handle: + :param sqlcipher_opts: options for the SQLCipherDatabase + :type sqlcipher_opts: SQLCipherOpts instance + """ + # XXX assert CryptoOptions + opts = sqlcipher_opts + _set_key(db_handle, opts.key, opts.is_raw_key) + _set_cipher(db_handle, opts.cipher) + _set_kdf_iter(db_handle, opts.kdf_iter) + _set_cipher_page_size(db_handle, opts.cipher_page_size) + + +def _set_key(db_handle, key, is_raw_key): + """ + Set the C{key} for use with the database. + + The process of creating a new, encrypted database is called 'keying' + the database. SQLCipher uses just-in-time key derivation at the point + it is first needed for an operation. This means that the key (and any + options) must be set before the first operation on the database. As + soon as the database is touched (e.g. SELECT, CREATE TABLE, UPDATE, + etc.) and pages need to be read or written, the key is prepared for + use. + + Implementation Notes: + + * PRAGMA key should generally be called as the first operation on a + database. + + :param key: The key for use with the database. + :type key: str + :param is_raw_key: Whether C{key} is a raw 64-char hex string or a + passphrase that should be hashed to obtain the + encyrption key. + :type is_raw_key: bool + """ + if is_raw_key: + _set_key_raw(db_handle, key) + else: + _set_key_passphrase(db_handle, key) + + +def _set_key_passphrase(cls, db_handle, passphrase): + """ + Set a passphrase for encryption key derivation. + + The key itself can be a passphrase, which is converted to a key using + PBKDF2 key derivation. The result is used as the encryption key for + the database. By using this method, there is no way to alter the KDF; + if you want to do so you should use a raw key instead and derive the + key using your own KDF. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param passphrase: The passphrase used to derive the encryption key. + :type passphrase: str + """ + db_handle.cursor().execute("PRAGMA key = '%s'" % passphrase) + + +def _set_key_raw(db_handle, key): + """ + Set a raw hexadecimal encryption key. + + It is possible to specify an exact byte sequence using a blob literal. + With this method, it is the calling application's responsibility to + ensure that the data provided is a 64 character hex string, which will + be converted directly to 32 bytes (256 bits) of key data. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param key: A 64 character hex string. + :type key: str + """ + if not all(c in string.hexdigits for c in key): + raise NotAnHexString(key) + db_handle.cursor().execute('PRAGMA key = "x\'%s"' % key) + + +def _set_cipher(db_handle, cipher='aes-256-cbc'): + """ + Set the cipher and mode to use for symmetric encryption. + + SQLCipher uses aes-256-cbc as the default cipher and mode of + operation. It is possible to change this, though not generally + recommended, using PRAGMA cipher. + + SQLCipher makes direct use of libssl, so all cipher options available + to libssl are also available for use with SQLCipher. See `man enc` for + OpenSSL's supported ciphers. + + Implementation Notes: + + * PRAGMA cipher must be called after PRAGMA key and before the first + actual database operation or it will have no effect. + + * If a non-default value is used PRAGMA cipher to create a database, + it must also be called every time that database is opened. + + * SQLCipher does not implement its own encryption. Instead it uses the + widely available and peer-reviewed OpenSSL libcrypto for all + cryptographic functions. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param cipher: The cipher and mode to use. + :type cipher: str + """ + db_handle.cursor().execute("PRAGMA cipher = '%s'" % cipher) + + +def _set_kdf_iter(db_handle, kdf_iter=4000): + """ + Set the number of iterations for the key derivation function. + + SQLCipher uses PBKDF2 key derivation to strengthen the key and make it + resistent to brute force and dictionary attacks. The default + configuration uses 4000 PBKDF2 iterations (effectively 16,000 SHA1 + operations). PRAGMA kdf_iter can be used to increase or decrease the + number of iterations used. + + Implementation Notes: + + * PRAGMA kdf_iter must be called after PRAGMA key and before the first + actual database operation or it will have no effect. + + * If a non-default value is used PRAGMA kdf_iter to create a database, + it must also be called every time that database is opened. + + * It is not recommended to reduce the number of iterations if a + passphrase is in use. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param kdf_iter: The number of iterations to use. + :type kdf_iter: int + """ + db_handle.cursor().execute("PRAGMA kdf_iter = '%d'" % kdf_iter) + + +def _set_cipher_page_size(db_handle, cipher_page_size=1024): + """ + Set the page size of the encrypted database. + + SQLCipher 2 introduced the new PRAGMA cipher_page_size that can be + used to adjust the page size for the encrypted database. The default + page size is 1024 bytes, but it can be desirable for some applications + to use a larger page size for increased performance. For instance, + some recent testing shows that increasing the page size can noticeably + improve performance (5-30%) for certain queries that manipulate a + large number of pages (e.g. selects without an index, large inserts in + a transaction, big deletes). + + To adjust the page size, call the pragma immediately after setting the + key for the first time and each subsequent time that you open the + database. + + Implementation Notes: + + * PRAGMA cipher_page_size must be called after PRAGMA key and before + the first actual database operation or it will have no effect. + + * If a non-default value is used PRAGMA cipher_page_size to create a + database, it must also be called every time that database is opened. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param cipher_page_size: The page size. + :type cipher_page_size: int + """ + db_handle.cursor().execute( + "PRAGMA cipher_page_size = '%d'" % cipher_page_size) + + +# XXX UNUSED ? +def set_rekey(db_handle, new_key, is_raw_key): + """ + Change the key of an existing encrypted database. + + To change the key on an existing encrypted database, it must first be + unlocked with the current encryption key. Once the database is + readable and writeable, PRAGMA rekey can be used to re-encrypt every + page in the database with a new key. + + * PRAGMA rekey must be called after PRAGMA key. It can be called at any + time once the database is readable. + + * PRAGMA rekey can not be used to encrypted a standard SQLite + database! It is only useful for changing the key on an existing + database. + + * Previous versions of SQLCipher provided a PRAGMA rekey_cipher and + code>PRAGMA rekey_kdf_iter. These are deprecated and should not be + used. Instead, use sqlcipher_export(). + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param new_key: The new key. + :type new_key: str + :param is_raw_key: Whether C{password} is a raw 64-char hex string or a + passphrase that should be hashed to obtain the encyrption + key. + :type is_raw_key: bool + """ + if is_raw_key: + _set_rekey_raw(db_handle, new_key) + else: + _set_rekey_passphrase(db_handle, new_key) + + +def _set_rekey_passphrase(db_handle, passphrase): + """ + Change the passphrase for encryption key derivation. + + The key itself can be a passphrase, which is converted to a key using + PBKDF2 key derivation. The result is used as the encryption key for + the database. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param passphrase: The passphrase used to derive the encryption key. + :type passphrase: str + """ + db_handle.cursor().execute("PRAGMA rekey = '%s'" % passphrase) + + +def _set_rekey_raw(cls, db_handle, key): + """ + Change the raw hexadecimal encryption key. + + It is possible to specify an exact byte sequence using a blob literal. + With this method, it is the calling application's responsibility to + ensure that the data provided is a 64 character hex string, which will + be converted directly to 32 bytes (256 bits) of key data. + + :param db_handle: A handle to the SQLCipher database. + :type db_handle: pysqlcipher.Connection + :param key: A 64 character hex string. + :type key: str + """ + if not all(c in string.hexdigits for c in key): + raise NotAnHexString(key) + db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % key) + + +def set_synchronous_off(db_handle): + """ + Change the setting of the "synchronous" flag to OFF. + """ + logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF") + db_handle.cursor().execute('PRAGMA synchronous=OFF') + + +def set_synchronous_normal(db_handle): + """ + Change the setting of the "synchronous" flag to NORMAL. + """ + logger.debug("SQLCIPHER: SETTING SYNCHRONOUS NORMAL") + db_handle.cursor().execute('PRAGMA synchronous=NORMAL') + + +def set_mem_temp_store(cls, db_handle): + """ + Use a in-memory store for temporary tables. + """ + logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY") + db_handle.cursor().execute('PRAGMA temp_store=MEMORY') + + +def set_write_ahead_logging(cls, db_handle): + """ + Enable write-ahead logging, and set the autocheckpoint to 50 pages. + + Setting the autocheckpoint to a small value, we make the reads not + suffer too much performance degradation. + + From the sqlite docs: + + "There is a tradeoff between average read performance and average write + performance. To maximize the read performance, one wants to keep the + WAL as small as possible and hence run checkpoints frequently, perhaps + as often as every COMMIT. To maximize write performance, one wants to + amortize the cost of each checkpoint over as many writes as possible, + meaning that one wants to run checkpoints infrequently and let the WAL + grow as large as possible before each checkpoint. The decision of how + often to run checkpoints may therefore vary from one application to + another depending on the relative read and write performance + requirements of the application. The default strategy is to run a + checkpoint once the WAL reaches 1000 pages" + """ + logger.debug("SQLCIPHER: SETTING WRITE-AHEAD LOGGING") + db_handle.cursor().execute('PRAGMA journal_mode=WAL') + # The optimum value can still use a little bit of tuning, but we favor + # small sizes of the WAL file to get fast reads, since we assume that + # the writes will be quick enough to not block too much. + + # TODO + # As a further improvement, we might want to set autocheckpoint to 0 + # here and do the checkpoints manually in a separate thread, to avoid + # any blocks in the main thread (we should run a loopingcall from here) + db_handle.cursor().execute('PRAGMA wal_autocheckpoint=50') + + +class NotAnHexString(Exception): + """ + Raised when trying to (raw) key the database with a non-hex string. + """ + pass diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 45629045..613903f7 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -44,7 +44,6 @@ handled by Soledad should be created by SQLCipher >= 2.0. import logging import multiprocessing import os -import string import threading import time import json @@ -64,6 +63,7 @@ from leap.soledad.client.target import SoledadSyncTarget from leap.soledad.client.target import PendingReceivedDocsSyncError from leap.soledad.client.sync import SoledadSynchronizer from leap.soledad.client.mp_safe_db import MPSafeSQLiteDB +from leap.soledad.client import pragmas from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument @@ -91,6 +91,55 @@ SQLITE_CHECK_SAME_THREAD = False SQLITE_ISOLATION_LEVEL = None +class SQLCipherOptions(object): + def __init__(self, path, key, create=True, is_raw_key=False, + cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, + document_factory=None, defer_encryption=False, + sync_db_key=None): + """ + Options for the initialization of an SQLCipher database. + + :param path: The filesystem path for the database to open. + :type path: str + :param create: + True/False, should the database be created if it doesn't + already exist? + :param create: bool + :param document_factory: + A function that will be called with the same parameters as + Document.__init__. + :type document_factory: callable + :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt + document contents when syncing. + :type crypto: soledad.crypto.SoledadCrypto + :param is_raw_key: + Whether ``password`` is a raw 64-char hex string or a passphrase + that should be hashed to obtain the encyrption key. + :type raw_key: bool + :param cipher: The cipher and mode to use. + :type cipher: str + :param kdf_iter: The number of iterations to use. + :type kdf_iter: int + :param cipher_page_size: The page size. + :type cipher_page_size: int + :param defer_encryption: + Whether to defer encryption/decryption of documents, or do it + inline while syncing. + :type defer_encryption: bool + """ + self.path = path + self.key = key + self.is_raw_key = is_raw_key + self.create = create + self.cipher = cipher + self.kdf_iter = kdf_iter + self.cipher_page_size = cipher_page_size + self.defer_encryption = defer_encryption + self.sync_db_key = sync_db_key + self.document_factory = None + + +# XXX Use SQLCIpherOptions instead def open(path, password, create=True, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, defer_encryption=False, sync_db_key=None): @@ -108,56 +157,22 @@ def open(path, password, create=True, document_factory=None, crypto=None, Will raise u1db.errors.DatabaseDoesNotExist if create=False and the database does not already exist. - :param path: The filesystem path for the database to open. - :type path: str - :param create: True/False, should the database be created if it doesn't - already exist? - :param create: bool - :param document_factory: A function that will be called with the same - parameters as Document.__init__. - :type document_factory: callable - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. - :type raw_key: bool - :param cipher: The cipher and mode to use. - :type cipher: str - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int - :param cipher_page_size: The page size. - :type cipher_page_size: int - :param defer_encryption: Whether to defer encryption/decryption of - documents, or do it inline while syncing. - :type defer_encryption: bool - :return: An instance of Database. :rtype SQLCipherDatabase """ - return SQLCipherDatabase.open_database( - path, password, create=create, document_factory=document_factory, - crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, - cipher_page_size=cipher_page_size, defer_encryption=defer_encryption, - sync_db_key=sync_db_key) - - -# -# Exceptions -# - -class DatabaseIsNotEncrypted(Exception): - """ - Exception raised when trying to open non-encrypted databases. - """ - pass - - -class NotAnHexString(Exception): - """ - Raised when trying to (raw) key the database with a non-hex string. - """ - pass + args = (path, password) + kwargs = { + 'create': create, + 'document_factory': document_factory, + 'crypto': crypto, + 'raw_key': raw_key, + 'cipher': cipher, + 'kdf_iter': kdf_iter, + 'cipher_page_size': cipher_page_size, + 'defer_encryption': defer_encryption, + 'sync_db_key': sync_db_key} + # XXX pass only a CryptoOptions object around + return SQLCipherDatabase.open_database(*args, **kwargs) # @@ -200,6 +215,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): same database replica. """ + # XXX Use SQLCIpherOptions instead def __init__(self, sqlcipher_file, password, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, sync_db_key=None): @@ -214,30 +230,10 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): experience several kinds of leakages. *** IMPORTANT *** - - :param sqlcipher_file: The path for the SQLCipher file. - :type sqlcipher_file: str - :param password: The password that protects the SQLCipher db. - :type password: str - :param document_factory: A function that will be called with the same - parameters as Document.__init__. - :type document_factory: callable - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto - :param raw_key: Whether password is a raw 64-char hex string or a - passphrase that should be hashed to obtain the - encyrption key. - :type raw_key: bool - :param cipher: The cipher and mode to use. - :type cipher: str - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int - :param cipher_page_size: The page size. - :type cipher_page_size: int """ # ensure the db is encrypted if the file already exists if os.path.exists(sqlcipher_file): + # XXX pass only a CryptoOptions object around self.assert_db_is_encrypted( sqlcipher_file, password, raw_key, cipher, kdf_iter, cipher_page_size) @@ -249,16 +245,19 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): isolation_level=SQLITE_ISOLATION_LEVEL, check_same_thread=SQLITE_CHECK_SAME_THREAD) # set SQLCipher cryptographic parameters - self._set_crypto_pragmas( + + # XXX allow optiona deferredChain here ? + pragmas.set_crypto_pragmas( self._db_handle, password, raw_key, cipher, kdf_iter, cipher_page_size) if os.environ.get('LEAP_SQLITE_NOSYNC'): - self._pragma_synchronous_off(self._db_handle) + pragmas.set_synchronous_off(self._db_handle) else: - self._pragma_synchronous_normal(self._db_handle) + pragmas.set_synchronous_normal(self._db_handle) if os.environ.get('LEAP_SQLITE_MEMSTORE'): - self._pragma_mem_temp_store(self._db_handle) - self._pragma_write_ahead_logging(self._db_handle) + pragmas.set_mem_temp_store(self._db_handle) + pragmas.set_write_ahead_logging(self._db_handle) + self._real_replica_uid = None self._ensure_schema() self._crypto = crypto @@ -296,6 +295,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self._syncers = {} @classmethod + # XXX Use SQLCIpherOptions instead def _open_database(cls, sqlcipher_file, password, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, @@ -303,29 +303,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ Open a SQLCipher database. - :param sqlcipher_file: The path for the SQLCipher file. - :type sqlcipher_file: str - :param password: The password that protects the SQLCipher db. - :type password: str - :param document_factory: A function that will be called with the same - parameters as Document.__init__. - :type document_factory: callable - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. - :type raw_key: bool - :param cipher: The cipher and mode to use. - :type cipher: str - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int - :param cipher_page_size: The page size. - :type cipher_page_size: int - :param defer_encryption: Whether to defer encryption/decryption of - documents, or do it inline while syncing. - :type defer_encryption: bool - :return: The database object. :rtype: SQLCipherDatabase """ @@ -346,7 +323,9 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): try: # set cryptographic params - cls._set_crypto_pragmas( + + # XXX pass only a CryptoOptions object around + pragmas.set_crypto_pragmas( db_handle, password, raw_key, cipher, kdf_iter, cipher_page_size) c = db_handle.cursor() @@ -372,11 +351,12 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): cipher_page_size=cipher_page_size, sync_db_key=sync_db_key) @classmethod - def open_database(cls, sqlcipher_file, password, create, backend_cls=None, + def open_database(cls, sqlcipher_file, password, create, document_factory=None, crypto=None, raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, defer_encryption=False, sync_db_key=None): + # XXX pass only a CryptoOptions object around """ Open a SQLCipher database. @@ -388,67 +368,29 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): *** IMPORTANT *** - :param sqlcipher_file: The path for the SQLCipher file. - :type sqlcipher_file: str - - :param password: The password that protects the SQLCipher db. - :type password: str - - :param create: Should the datbase be created if it does not already - exist? - :type create: bool - - :param backend_cls: A class to use as backend. - :type backend_cls: type - - :param document_factory: A function that will be called with the same - parameters as Document.__init__. - :type document_factory: callable - - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto - - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the - encyrption key. - :type raw_key: bool - - :param cipher: The cipher and mode to use. - :type cipher: str - - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int - - :param cipher_page_size: The page size. - :type cipher_page_size: int - - :param defer_encryption: Whether to defer encryption/decryption of - documents, or do it inline while syncing. - :type defer_encryption: bool - :return: The database object. :rtype: SQLCipherDatabase """ cls.defer_encryption = defer_encryption + args = sqlcipher_file, password + kwargs = { + 'crypto': crypto, + 'raw_key': raw_key, + 'cipher': cipher, + 'kdf_iter': kdf_iter, + 'cipher_page_size': cipher_page_size, + 'defer_encryption': defer_encryption, + 'sync_db_key': sync_db_key, + 'document_factory': document_factory, + } try: - return cls._open_database( - sqlcipher_file, password, document_factory=document_factory, - crypto=crypto, raw_key=raw_key, cipher=cipher, - kdf_iter=kdf_iter, cipher_page_size=cipher_page_size, - defer_encryption=defer_encryption, sync_db_key=sync_db_key) + return cls._open_database(*args, **kwargs) except u1db_errors.DatabaseDoesNotExist: if not create: raise - # TODO: remove backend class from here. - if backend_cls is None: - # default is SQLCipherPartialExpandDatabase - backend_cls = SQLCipherDatabase - return backend_cls( - sqlcipher_file, password, document_factory=document_factory, - crypto=crypto, raw_key=raw_key, cipher=cipher, - kdf_iter=kdf_iter, cipher_page_size=cipher_page_size, - sync_db_key=sync_db_key) + + # XXX here we were missing sync_db_key, intentional? + return SQLCipherDatabase(*args, **kwargs) def sync(self, url, creds=None, autocreate=True, defer_decryption=True): """ @@ -592,7 +534,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self._sync_db = MPSafeSQLiteDB(sync_db_path) # protect the sync db with a password if self._sync_db_key is not None: - self._set_crypto_pragmas( + # XXX pass only a CryptoOptions object around + pragmas.set_crypto_pragmas( self._sync_db, self._sync_db_key, False, 'aes-256-cbc', 4000, 1024) self._sync_db_write_lock = threading.Lock() @@ -712,6 +655,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # SQLCipher API methods # + # XXX Use SQLCIpherOptions instead @classmethod def assert_db_is_encrypted(cls, sqlcipher_file, key, raw_key, cipher, kdf_iter, cipher_page_size): @@ -755,314 +699,12 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): sqlcipher_file, isolation_level=SQLITE_ISOLATION_LEVEL, check_same_thread=SQLITE_CHECK_SAME_THREAD) - cls._set_crypto_pragmas( + pragmas.set_crypto_pragmas( db_handle, key, raw_key, cipher, kdf_iter, cipher_page_size) db_handle.cursor().execute( 'SELECT count(*) FROM sqlite_master') - @classmethod - def _set_crypto_pragmas(cls, db_handle, key, raw_key, cipher, kdf_iter, - cipher_page_size): - """ - Set cryptographic params (key, cipher, KDF number of iterations and - cipher page size). - """ - cls._pragma_key(db_handle, key, raw_key) - cls._pragma_cipher(db_handle, cipher) - cls._pragma_kdf_iter(db_handle, kdf_iter) - cls._pragma_cipher_page_size(db_handle, cipher_page_size) - - @classmethod - def _pragma_key(cls, db_handle, key, raw_key): - """ - Set the C{key} for use with the database. - - The process of creating a new, encrypted database is called 'keying' - the database. SQLCipher uses just-in-time key derivation at the point - it is first needed for an operation. This means that the key (and any - options) must be set before the first operation on the database. As - soon as the database is touched (e.g. SELECT, CREATE TABLE, UPDATE, - etc.) and pages need to be read or written, the key is prepared for - use. - - Implementation Notes: - - * PRAGMA key should generally be called as the first operation on a - database. - - :param key: The key for use with the database. - :type key: str - :param raw_key: Whether C{key} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. - :type raw_key: bool - """ - if raw_key: - cls._pragma_key_raw(db_handle, key) - else: - cls._pragma_key_passphrase(db_handle, key) - - @classmethod - def _pragma_key_passphrase(cls, db_handle, passphrase): - """ - Set a passphrase for encryption key derivation. - - The key itself can be a passphrase, which is converted to a key using - PBKDF2 key derivation. The result is used as the encryption key for - the database. By using this method, there is no way to alter the KDF; - if you want to do so you should use a raw key instead and derive the - key using your own KDF. - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param passphrase: The passphrase used to derive the encryption key. - :type passphrase: str - """ - db_handle.cursor().execute("PRAGMA key = '%s'" % passphrase) - - @classmethod - def _pragma_key_raw(cls, db_handle, key): - """ - Set a raw hexadecimal encryption key. - - It is possible to specify an exact byte sequence using a blob literal. - With this method, it is the calling application's responsibility to - ensure that the data provided is a 64 character hex string, which will - be converted directly to 32 bytes (256 bits) of key data. - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param key: A 64 character hex string. - :type key: str - """ - if not all(c in string.hexdigits for c in key): - raise NotAnHexString(key) - db_handle.cursor().execute('PRAGMA key = "x\'%s"' % key) - - @classmethod - def _pragma_cipher(cls, db_handle, cipher='aes-256-cbc'): - """ - Set the cipher and mode to use for symmetric encryption. - - SQLCipher uses aes-256-cbc as the default cipher and mode of - operation. It is possible to change this, though not generally - recommended, using PRAGMA cipher. - - SQLCipher makes direct use of libssl, so all cipher options available - to libssl are also available for use with SQLCipher. See `man enc` for - OpenSSL's supported ciphers. - - Implementation Notes: - - * PRAGMA cipher must be called after PRAGMA key and before the first - actual database operation or it will have no effect. - - * If a non-default value is used PRAGMA cipher to create a database, - it must also be called every time that database is opened. - - * SQLCipher does not implement its own encryption. Instead it uses the - widely available and peer-reviewed OpenSSL libcrypto for all - cryptographic functions. - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param cipher: The cipher and mode to use. - :type cipher: str - """ - db_handle.cursor().execute("PRAGMA cipher = '%s'" % cipher) - - @classmethod - def _pragma_kdf_iter(cls, db_handle, kdf_iter=4000): - """ - Set the number of iterations for the key derivation function. - - SQLCipher uses PBKDF2 key derivation to strengthen the key and make it - resistent to brute force and dictionary attacks. The default - configuration uses 4000 PBKDF2 iterations (effectively 16,000 SHA1 - operations). PRAGMA kdf_iter can be used to increase or decrease the - number of iterations used. - - Implementation Notes: - - * PRAGMA kdf_iter must be called after PRAGMA key and before the first - actual database operation or it will have no effect. - - * If a non-default value is used PRAGMA kdf_iter to create a database, - it must also be called every time that database is opened. - - * It is not recommended to reduce the number of iterations if a - passphrase is in use. - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int - """ - db_handle.cursor().execute("PRAGMA kdf_iter = '%d'" % kdf_iter) - - @classmethod - def _pragma_cipher_page_size(cls, db_handle, cipher_page_size=1024): - """ - Set the page size of the encrypted database. - - SQLCipher 2 introduced the new PRAGMA cipher_page_size that can be - used to adjust the page size for the encrypted database. The default - page size is 1024 bytes, but it can be desirable for some applications - to use a larger page size for increased performance. For instance, - some recent testing shows that increasing the page size can noticeably - improve performance (5-30%) for certain queries that manipulate a - large number of pages (e.g. selects without an index, large inserts in - a transaction, big deletes). - - To adjust the page size, call the pragma immediately after setting the - key for the first time and each subsequent time that you open the - database. - - Implementation Notes: - - * PRAGMA cipher_page_size must be called after PRAGMA key and before - the first actual database operation or it will have no effect. - - * If a non-default value is used PRAGMA cipher_page_size to create a - database, it must also be called every time that database is opened. - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param cipher_page_size: The page size. - :type cipher_page_size: int - """ - db_handle.cursor().execute( - "PRAGMA cipher_page_size = '%d'" % cipher_page_size) - - @classmethod - def _pragma_rekey(cls, db_handle, new_key, raw_key): - """ - Change the key of an existing encrypted database. - - To change the key on an existing encrypted database, it must first be - unlocked with the current encryption key. Once the database is - readable and writeable, PRAGMA rekey can be used to re-encrypt every - page in the database with a new key. - - * PRAGMA rekey must be called after PRAGMA key. It can be called at any - time once the database is readable. - - * PRAGMA rekey can not be used to encrypted a standard SQLite - database! It is only useful for changing the key on an existing - database. - - * Previous versions of SQLCipher provided a PRAGMA rekey_cipher and - code>PRAGMA rekey_kdf_iter. These are deprecated and should not be - used. Instead, use sqlcipher_export(). - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param new_key: The new key. - :type new_key: str - :param raw_key: Whether C{password} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. - :type raw_key: bool - """ - # XXX change key param! - if raw_key: - cls._pragma_rekey_raw(db_handle, key) - else: - cls._pragma_rekey_passphrase(db_handle, key) - - @classmethod - def _pragma_rekey_passphrase(cls, db_handle, passphrase): - """ - Change the passphrase for encryption key derivation. - - The key itself can be a passphrase, which is converted to a key using - PBKDF2 key derivation. The result is used as the encryption key for - the database. - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param passphrase: The passphrase used to derive the encryption key. - :type passphrase: str - """ - db_handle.cursor().execute("PRAGMA rekey = '%s'" % passphrase) - - @classmethod - def _pragma_rekey_raw(cls, db_handle, key): - """ - Change the raw hexadecimal encryption key. - - It is possible to specify an exact byte sequence using a blob literal. - With this method, it is the calling application's responsibility to - ensure that the data provided is a 64 character hex string, which will - be converted directly to 32 bytes (256 bits) of key data. - - :param db_handle: A handle to the SQLCipher database. - :type db_handle: pysqlcipher.Connection - :param key: A 64 character hex string. - :type key: str - """ - if not all(c in string.hexdigits for c in key): - raise NotAnHexString(key) - # XXX change passphrase param! - db_handle.cursor().execute('PRAGMA rekey = "x\'%s"' % passphrase) - - @classmethod - def _pragma_synchronous_off(cls, db_handle): - """ - Change the setting of the "synchronous" flag to OFF. - """ - logger.debug("SQLCIPHER: SETTING SYNCHRONOUS OFF") - db_handle.cursor().execute('PRAGMA synchronous=OFF') - - @classmethod - def _pragma_synchronous_normal(cls, db_handle): - """ - Change the setting of the "synchronous" flag to NORMAL. - """ - logger.debug("SQLCIPHER: SETTING SYNCHRONOUS NORMAL") - db_handle.cursor().execute('PRAGMA synchronous=NORMAL') - - @classmethod - def _pragma_mem_temp_store(cls, db_handle): - """ - Use a in-memory store for temporary tables. - """ - logger.debug("SQLCIPHER: SETTING TEMP_STORE MEMORY") - db_handle.cursor().execute('PRAGMA temp_store=MEMORY') - - @classmethod - def _pragma_write_ahead_logging(cls, db_handle): - """ - Enable write-ahead logging, and set the autocheckpoint to 50 pages. - - Setting the autocheckpoint to a small value, we make the reads not - suffer too much performance degradation. - - From the sqlite docs: - - "There is a tradeoff between average read performance and average write - performance. To maximize the read performance, one wants to keep the - WAL as small as possible and hence run checkpoints frequently, perhaps - as often as every COMMIT. To maximize write performance, one wants to - amortize the cost of each checkpoint over as many writes as possible, - meaning that one wants to run checkpoints infrequently and let the WAL - grow as large as possible before each checkpoint. The decision of how - often to run checkpoints may therefore vary from one application to - another depending on the relative read and write performance - requirements of the application. The default strategy is to run a - checkpoint once the WAL reaches 1000 pages" - """ - logger.debug("SQLCIPHER: SETTING WRITE-AHEAD LOGGING") - db_handle.cursor().execute('PRAGMA journal_mode=WAL') - # The optimum value can still use a little bit of tuning, but we favor - # small sizes of the WAL file to get fast reads, since we assume that - # the writes will be quick enough to not block too much. - - # TODO - # As a further improvement, we might want to set autocheckpoint to 0 - # here and do the checkpoints manually in a separate thread, to avoid - # any blocks in the main thread (we should run a loopingcall from here) - db_handle.cursor().execute('PRAGMA wal_autocheckpoint=50') - # Extra query methods: extensions to the base sqlite implmentation. def get_count_from_index(self, index_name, *key_values): @@ -1162,5 +804,16 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): def replica_uid(self): return self._get_replica_uid() +# +# Exceptions +# + + +class DatabaseIsNotEncrypted(Exception): + """ + Exception raised when trying to open non-encrypted databases. + """ + pass + sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase) -- cgit v1.2.3 From 9c56adfd27e96c44c12ad5295c42e6b8d9bcad98 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 22 Sep 2014 20:03:45 -0500 Subject: move public api to its own file --- client/src/leap/soledad/client/__init__.py | 824 +-------------------- client/src/leap/soledad/client/api.py | 822 ++++++++++++++++++++ client/src/leap/soledad/client/mp_safe_db.py | 112 --- .../src/leap/soledad/client/mp_safe_db_TOREMOVE.py | 112 +++ client/src/leap/soledad/client/sqlcipher.py | 323 ++++---- client/src/leap/soledad/client/sync.py | 4 - 6 files changed, 1107 insertions(+), 1090 deletions(-) create mode 100644 client/src/leap/soledad/client/api.py delete mode 100644 client/src/leap/soledad/client/mp_safe_db.py create mode 100644 client/src/leap/soledad/client/mp_safe_db_TOREMOVE.py diff --git a/client/src/leap/soledad/client/__init__.py b/client/src/leap/soledad/client/__init__.py index 50fcff2c..245a8971 100644 --- a/client/src/leap/soledad/client/__init__.py +++ b/client/src/leap/soledad/client/__init__.py @@ -16,828 +16,12 @@ # along with this program. If not, see . """ Soledad - Synchronization Of Locally Encrypted Data Among Devices. - -Soledad is the part of LEAP that manages storage and synchronization of -application data. It is built on top of U1DB reference Python API and -implements (1) a SQLCipher backend for local storage in the client, (2) a -SyncTarget that encrypts data before syncing, and (3) a CouchDB backend for -remote storage in the server side. -""" -import binascii -import errno -import httplib -import logging -import os -import socket -import ssl -import urlparse - - -try: - import cchardet as chardet -except ImportError: - import chardet - -from u1db.remote import http_client -from u1db.remote.ssl_match_hostname import match_hostname - -from leap.common.config import get_path_prefix -from leap.soledad.common import ( - SHARED_DB_NAME, - soledad_assert, - soledad_assert_type -) -from leap.soledad.client.events import ( - SOLEDAD_NEW_DATA_TO_SYNC, - SOLEDAD_DONE_DATA_SYNC, - signal, -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.crypto import SoledadCrypto -from leap.soledad.client.secrets import SoledadSecrets -from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client.sqlcipher import open as sqlcipher_open -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.target import SoledadSyncTarget - - -logger = logging.getLogger(name=__name__) - - -# -# Constants -# - -SOLEDAD_CERT = None """ -Path to the certificate file used to certify the SSL connection between -Soledad client and server. -""" - - -# -# Soledad: local encrypted storage and remote encrypted sync. -# - -class Soledad(object): - """ - Soledad provides encrypted data storage and sync. - - A Soledad instance is used to store and retrieve data in a local encrypted - database and synchronize this database with Soledad server. - - This class is also responsible for bootstrapping users' account by - creating cryptographic secrets and/or storing/fetching them on Soledad - server. - - Soledad uses C{leap.common.events} to signal events. The possible events - to be signaled are: - - SOLEDAD_CREATING_KEYS: emitted during bootstrap sequence when key - generation starts. - SOLEDAD_DONE_CREATING_KEYS: emitted during bootstrap sequence when key - generation finishes. - SOLEDAD_UPLOADING_KEYS: emitted during bootstrap sequence when soledad - starts sending keys to server. - SOLEDAD_DONE_UPLOADING_KEYS: emitted during bootstrap sequence when - soledad finishes sending keys to server. - SOLEDAD_DOWNLOADING_KEYS: emitted during bootstrap sequence when - soledad starts to retrieve keys from server. - SOLEDAD_DONE_DOWNLOADING_KEYS: emitted during bootstrap sequence when - soledad finishes downloading keys from server. - SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when - there's indeed new data to be synchronized between local database - replica and server's replica. - SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has - finished synchronizing with remote replica. - """ - - LOCAL_DATABASE_FILE_NAME = 'soledad.u1db' - """ - The name of the local SQLCipher U1DB database file. - """ - - STORAGE_SECRETS_FILE_NAME = "soledad.json" - """ - The name of the file where the storage secrets will be stored. - """ - - DEFAULT_PREFIX = os.path.join(get_path_prefix(), 'leap', 'soledad') - """ - Prefix for default values for path. - """ - - def __init__(self, uuid, passphrase, secrets_path, local_db_path, - server_url, cert_file, - auth_token=None, secret_id=None, defer_encryption=False): - """ - Initialize configuration, cryptographic keys and dbs. - - :param uuid: User's uuid. - :type uuid: str - - :param passphrase: The passphrase for locking and unlocking encryption - secrets for local and remote storage. - :type passphrase: unicode - - :param secrets_path: Path for storing encrypted key used for - symmetric encryption. - :type secrets_path: str - - :param local_db_path: Path for local encrypted storage db. - :type local_db_path: str - - :param server_url: URL for Soledad server. This is used either to sync - with the user's remote db and to interact with the - shared recovery database. - :type server_url: str - - :param cert_file: Path to the certificate of the ca used - to validate the SSL certificate used by the remote - soledad server. - :type cert_file: str - - :param auth_token: Authorization token for accessing remote databases. - :type auth_token: str - - :param secret_id: The id of the storage secret to be used. - :type secret_id: str - - :param defer_encryption: Whether to defer encryption/decryption of - documents, or do it inline while syncing. - :type defer_encryption: bool - - :raise BootstrapSequenceError: Raised when the secret generation and - storage on server sequence has failed - for some reason. - """ - # store config params - self._uuid = uuid - self._passphrase = passphrase - self._secrets_path = secrets_path - self._local_db_path = local_db_path - self._server_url = server_url - # configure SSL certificate - global SOLEDAD_CERT - SOLEDAD_CERT = cert_file - self._set_token(auth_token) - self._defer_encryption = defer_encryption - - self._init_config() - self._init_dirs() - - # init crypto variables - self._shared_db_instance = None - self._crypto = SoledadCrypto(self) - self._secrets = SoledadSecrets( - self._uuid, - self._passphrase, - self._secrets_path, - self._shared_db, - self._crypto, - secret_id=secret_id) - - # initiate bootstrap sequence - self._bootstrap() # might raise BootstrapSequenceError() - - def _init_config(self): - """ - Initialize configuration using default values for missing params. - """ - soledad_assert_type(self._passphrase, unicode) - # initialize secrets_path - if self._secrets_path is None: - self._secrets_path = os.path.join( - self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME) - # initialize local_db_path - if self._local_db_path is None: - self._local_db_path = os.path.join( - self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME) - # initialize server_url - soledad_assert( - self._server_url is not None, - 'Missing URL for Soledad server.') - - # - # initialization/destruction methods - # - - def _bootstrap(self): - """ - Bootstrap local Soledad instance. - - :raise BootstrapSequenceError: Raised when the secret generation and - storage on server sequence has failed for some reason. - """ - try: - self._secrets.bootstrap() - self._init_db() - except: - raise - - def _init_dirs(self): - """ - Create work directories. - - :raise OSError: in case file exists and is not a dir. - """ - paths = map( - lambda x: os.path.dirname(x), - [self._local_db_path, self._secrets_path]) - for path in paths: - try: - if not os.path.isdir(path): - logger.info('Creating directory: %s.' % path) - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - def _init_db(self): - """ - Initialize the U1DB SQLCipher database for local storage. - - Currently, Soledad uses the default SQLCipher cipher, i.e. - 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key and - uses the 'raw PRAGMA key' format to handle the key to SQLCipher. - """ - key = self._secrets.get_local_storage_key() - sync_db_key = self._secrets.get_sync_db_key() - self._db = sqlcipher_open( - self._local_db_path, - binascii.b2a_hex(key), # sqlcipher only accepts the hex version - create=True, - document_factory=SoledadDocument, - crypto=self._crypto, - raw_key=True, - defer_encryption=self._defer_encryption, - sync_db_key=binascii.b2a_hex(sync_db_key)) - - def close(self): - """ - Close underlying U1DB database. - """ - logger.debug("Closing soledad") - if hasattr(self, '_db') and isinstance( - self._db, - SQLCipherDatabase): - self._db.stop_sync() - self._db.close() - - @property - def _shared_db(self): - """ - Return an instance of the shared recovery database object. - - :return: The shared database. - :rtype: SoledadSharedDatabase - """ - if self._shared_db_instance is None: - self._shared_db_instance = SoledadSharedDatabase.open_database( - urlparse.urljoin(self.server_url, SHARED_DB_NAME), - self._uuid, - False, # db should exist at this point. - creds=self._creds) - return self._shared_db_instance - - # - # Document storage, retrieval and sync. - # - - def put_doc(self, doc): - """ - Update a document in the local encrypted database. - - ============================== WARNING ============================== - This method converts the document's contents to unicode in-place. This - means that after calling C{put_doc(doc)}, the contents of the - document, i.e. C{doc.content}, might be different from before the - call. - ============================== WARNING ============================== - - :param doc: the document to update - :type doc: SoledadDocument - - :return: the new revision identifier for the document - :rtype: str - """ - doc.content = self._convert_to_unicode(doc.content) - return self._db.put_doc(doc) - - def delete_doc(self, doc): - """ - Delete a document from the local encrypted database. - - :param doc: the document to delete - :type doc: SoledadDocument - - :return: the new revision identifier for the document - :rtype: str - """ - return self._db.delete_doc(doc) - - def get_doc(self, doc_id, include_deleted=False): - """ - Retrieve a document from the local encrypted database. - - :param doc_id: the unique document identifier - :type doc_id: str - :param include_deleted: if True, deleted documents will be - returned with empty content; otherwise asking - for a deleted document will return None - :type include_deleted: bool - - :return: the document object or None - :rtype: SoledadDocument - """ - return self._db.get_doc(doc_id, include_deleted=include_deleted) - - def get_docs(self, doc_ids, check_for_conflicts=True, - include_deleted=False): - """ - Get the content for many documents. - - :param doc_ids: a list of document identifiers - :type doc_ids: list - :param check_for_conflicts: if set False, then the conflict check will - be skipped, and 'None' will be returned instead of True/False - :type check_for_conflicts: bool - - :return: iterable giving the Document object for each document id - in matching doc_ids order. - :rtype: generator - """ - return self._db.get_docs( - doc_ids, check_for_conflicts=check_for_conflicts, - include_deleted=include_deleted) - - def get_all_docs(self, include_deleted=False): - """ - Get the JSON content for all documents in the database. - - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted - documents will not be included in the results. - :return: (generation, [Document]) - The current generation of the database, followed by a list of - all the documents in the database. - """ - return self._db.get_all_docs(include_deleted) - - def _convert_to_unicode(self, content): - """ - Converts content to unicode (or all the strings in content) - - NOTE: Even though this method supports any type, it will - currently ignore contents of lists, tuple or any other - iterable than dict. We don't need support for these at the - moment - - :param content: content to convert - :type content: object - - :rtype: object - """ - if isinstance(content, unicode): - return content - elif isinstance(content, str): - result = chardet.detect(content) - default = "utf-8" - encoding = result["encoding"] or default - try: - content = content.decode(encoding) - except UnicodeError as e: - logger.error("Unicode error: {0!r}. Using 'replace'".format(e)) - content = content.decode(encoding, 'replace') - return content - else: - if isinstance(content, dict): - for key in content.keys(): - content[key] = self._convert_to_unicode(content[key]) - return content - - def create_doc(self, content, doc_id=None): - """ - Create a new document in the local encrypted database. - - :param content: the contents of the new document - :type content: dict - :param doc_id: an optional identifier specifying the document id - :type doc_id: str - - :return: the new document - :rtype: SoledadDocument - """ - return self._db.create_doc( - self._convert_to_unicode(content), doc_id=doc_id) - - def create_doc_from_json(self, json, doc_id=None): - """ - Create a new document. - - You can optionally specify the document identifier, but the document - must not already exist. See 'put_doc' if you want to override an - existing document. - If the database specifies a maximum document size and the document - exceeds it, create will fail and raise a DocumentTooBig exception. - - :param json: The JSON document string - :type json: str - :param doc_id: An optional identifier specifying the document id. - :type doc_id: - :return: The new document - :rtype: SoledadDocument - """ - return self._db.create_doc_from_json(json, doc_id=doc_id) - - def create_index(self, index_name, *index_expressions): - """ - Create an named index, which can then be queried for future lookups. - Creating an index which already exists is not an error, and is cheap. - Creating an index which does not match the index_expressions of the - existing index is an error. - Creating an index will block until the expressions have been evaluated - and the index generated. - - :param index_name: A unique name which can be used as a key prefix - :type index_name: str - :param index_expressions: index expressions defining the index - information. - :type index_expressions: dict - - Examples: - - "fieldname", or "fieldname.subfieldname" to index alphabetically - sorted on the contents of a field. - - "number(fieldname, width)", "lower(fieldname)" - """ - if self._db: - return self._db.create_index( - index_name, *index_expressions) - - def delete_index(self, index_name): - """ - Remove a named index. - - :param index_name: The name of the index we are removing - :type index_name: str - """ - if self._db: - return self._db.delete_index(index_name) - - def list_indexes(self): - """ - List the definitions of all known indexes. - - :return: A list of [('index-name', ['field', 'field2'])] definitions. - :rtype: list - """ - if self._db: - return self._db.list_indexes() - - def get_from_index(self, index_name, *key_values): - """ - Return documents that match the keys supplied. - - You must supply exactly the same number of values as have been defined - in the index. It is possible to do a prefix match by using '*' to - indicate a wildcard match. You can only supply '*' to trailing entries, - (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) - It is also possible to append a '*' to the last supplied value (eg - 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - - :param index_name: The index to query - :type index_name: str - :param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - :type key_values: tuple - :return: List of [Document] - :rtype: list - """ - if self._db: - return self._db.get_from_index(index_name, *key_values) - - def get_count_from_index(self, index_name, *key_values): - """ - Return the count of the documents that match the keys and - values supplied. - - :param index_name: The index to query - :type index_name: str - :param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - :type key_values: tuple - :return: count. - :rtype: int - """ - if self._db: - return self._db.get_count_from_index(index_name, *key_values) - - def get_range_from_index(self, index_name, start_value, end_value): - """ - Return documents that fall within the specified range. - - Both ends of the range are inclusive. For both start_value and - end_value, one must supply exactly the same number of values as have - been defined in the index, or pass None. In case of a single column - index, a string is accepted as an alternative for a tuple with a single - value. It is possible to do a prefix match by using '*' to indicate - a wildcard match. You can only supply '*' to trailing entries, (eg - 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also - possible to append a '*' to the last supplied value (eg 'val*', '*', - '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - - :param index_name: The index to query - :type index_name: str - :param start_values: tuples of values that define the lower bound of - the range. eg, if you have an index with 3 fields then you would - have: (val1, val2, val3) - :type start_values: tuple - :param end_values: tuples of values that define the upper bound of the - range. eg, if you have an index with 3 fields then you would have: - (val1, val2, val3) - :type end_values: tuple - :return: List of [Document] - :rtype: list - """ - if self._db: - return self._db.get_range_from_index( - index_name, start_value, end_value) - - def get_index_keys(self, index_name): - """ - Return all keys under which documents are indexed in this index. - - :param index_name: The index to query - :type index_name: str - :return: [] A list of tuples of indexed keys. - :rtype: list - """ - if self._db: - return self._db.get_index_keys(index_name) - - def get_doc_conflicts(self, doc_id): - """ - Get the list of conflicts for the given document. - - :param doc_id: the document id - :type doc_id: str - - :return: a list of the document entries that are conflicted - :rtype: list - """ - if self._db: - return self._db.get_doc_conflicts(doc_id) - - def resolve_doc(self, doc, conflicted_doc_revs): - """ - Mark a document as no longer conflicted. - - :param doc: a document with the new content to be inserted. - :type doc: SoledadDocument - :param conflicted_doc_revs: a list of revisions that the new content - supersedes. - :type conflicted_doc_revs: list - """ - if self._db: - return self._db.resolve_doc(doc, conflicted_doc_revs) - - def sync(self, defer_decryption=True): - """ - Synchronize the local encrypted replica with a remote replica. - - This method blocks until a syncing lock is acquired, so there are no - attempts of concurrent syncs from the same client replica. - - :param url: the url of the target replica to sync with - :type url: str - - :param defer_decryption: Whether to defer the decryption process using - the intermediate database. If False, - decryption will be done inline. - :type defer_decryption: bool - - :return: The local generation before the synchronisation was - performed. - :rtype: str - """ - if self._db: - try: - local_gen = self._db.sync( - urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=False, - defer_decryption=defer_decryption) - signal(SOLEDAD_DONE_DATA_SYNC, self._uuid) - return local_gen - except Exception as e: - logger.error("Soledad exception when syncing: %s" % str(e)) - - def stop_sync(self): - """ - Stop the current syncing process. - """ - if self._db: - self._db.stop_sync() - - def need_sync(self, url): - """ - Return if local db replica differs from remote url's replica. - - :param url: The remote replica to compare with local replica. - :type url: str - - :return: Whether remote replica and local replica differ. - :rtype: bool - """ - target = SoledadSyncTarget( - url, self._db._get_replica_uid(), creds=self._creds, - crypto=self._crypto) - info = target.get_sync_info(self._db._get_replica_uid()) - # compare source generation with target's last known source generation - if self._db._get_generation() != info[4]: - signal(SOLEDAD_NEW_DATA_TO_SYNC, self._uuid) - return True - return False - - @property - def syncing(self): - """ - Property, True if the syncer is syncing. - """ - return self._db.syncing - - def _set_token(self, token): - """ - Set the authentication token for remote database access. - - Build the credentials dictionary with the following format: - - self._{ - 'token': { - 'uuid': '' - 'token': '' - } - - :param token: The authentication token. - :type token: str - """ - self._creds = { - 'token': { - 'uuid': self._uuid, - 'token': token, - } - } - - def _get_token(self): - """ - Return current token from credentials dictionary. - """ - return self._creds['token']['token'] - - token = property(_get_token, _set_token, doc='The authentication Token.') - - # - # Setters/getters - # - - def _get_uuid(self): - return self._uuid - - uuid = property(_get_uuid, doc='The user uuid.') - - def get_secret_id(self): - return self._secrets.secret_id - - def set_secret_id(self, secret_id): - self._secrets.set_secret_id(secret_id) - - secret_id = property( - get_secret_id, - set_secret_id, - doc='The active secret id.') - - def _set_secrets_path(self, secrets_path): - self._secrets.secrets_path = secrets_path - - def _get_secrets_path(self): - return self._secrets.secrets_path - - secrets_path = property( - _get_secrets_path, - _set_secrets_path, - doc='The path for the file containing the encrypted symmetric secret.') - - def _get_local_db_path(self): - return self._local_db_path - - local_db_path = property( - _get_local_db_path, - doc='The path for the local database replica.') - - def _get_server_url(self): - return self._server_url - - server_url = property( - _get_server_url, - doc='The URL of the Soledad server.') - - @property - def storage_secret(self): - """ - Return the secret used for symmetric encryption. - """ - return self._secrets.storage_secret - - @property - def remote_storage_secret(self): - """ - Return the secret used for encryption of remotely stored data. - """ - return self._secrets.remote_storage_secret - - @property - def secrets(self): - return self._secrets - - @property - def passphrase(self): - return self._secrets.passphrase - - def change_passphrase(self, new_passphrase): - """ - Change the passphrase that encrypts the storage secret. - - :param new_passphrase: The new passphrase. - :type new_passphrase: unicode - - :raise NoStorageSecret: Raised if there's no storage secret available. - """ - self._secrets.change_passphrase(new_passphrase) - - -# ---------------------------------------------------------------------------- -# Monkey patching u1db to be able to provide a custom SSL cert -# ---------------------------------------------------------------------------- - -# We need a more reasonable timeout (in seconds) -SOLEDAD_TIMEOUT = 120 - - -class VerifiedHTTPSConnection(httplib.HTTPSConnection): - """ - HTTPSConnection verifying server side certificates. - """ - # derived from httplib.py - - def connect(self): - """ - Connect to a host on a given (SSL) port. - """ - try: - source = self.source_address - sock = socket.create_connection((self.host, self.port), - SOLEDAD_TIMEOUT, source) - except AttributeError: - # source_address was introduced in 2.7 - sock = socket.create_connection((self.host, self.port), - SOLEDAD_TIMEOUT) - if self._tunnel_host: - self.sock = sock - self._tunnel() - - highest_supported = ssl.PROTOCOL_SSLv23 - - try: - # needs python 2.7.9+ - # negotiate the best available version, - # but explicitely disabled bad ones. - ctx = ssl.SSLContext(highest_supported) - ctx.options |= ssl.OP_NO_SSLv2 - ctx.options |= ssl.OP_NO_SSLv3 - - ctx.load_verify_locations(cafile=SOLEDAD_CERT) - ctx.verify_mode = ssl.CERT_REQUIRED - self.sock = ctx.wrap_socket(sock) - - except AttributeError: - self.sock = ssl.wrap_socket( - sock, ca_certs=SOLEDAD_CERT, cert_reqs=ssl.CERT_REQUIRED, - ssl_version=highest_supported) - - match_hostname(self.sock.getpeercert(), self.host) - - -old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection -http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection - - -__all__ = ['soledad_assert', 'Soledad'] +from leap.soledad.client.api import Soledad +from leap.soledad.common import soledad_assert from ._version import get_versions __version__ = get_versions()['version'] del get_versions + +__all__ = ['soledad_assert', 'Soledad', '__version__'] diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py new file mode 100644 index 00000000..bfb6c703 --- /dev/null +++ b/client/src/leap/soledad/client/api.py @@ -0,0 +1,822 @@ +# -*- coding: utf-8 -*- +# api.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Soledad - Synchronization Of Locally Encrypted Data Among Devices. + +This module holds the public api for Soledad. + +Soledad is the part of LEAP that manages storage and synchronization of +application data. It is built on top of U1DB reference Python API and +implements (1) a SQLCipher backend for local storage in the client, (2) a +SyncTarget that encrypts data before syncing, and (3) a CouchDB backend for +remote storage in the server side. +""" +import binascii +import errno +import httplib +import logging +import os +import socket +import ssl +import urlparse + + +try: + import cchardet as chardet +except ImportError: + import chardet + +from u1db.remote import http_client +from u1db.remote.ssl_match_hostname import match_hostname + +from leap.common.config import get_path_prefix +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common import soledad_assert +from leap.soledad.common import soledad_assert_type +from leap.soledad.common.document import SoledadDocument + +from leap.soledad.client import events as soledad_events +from leap.soledad.client.crypto import SoledadCrypto +from leap.soledad.client.secrets import SoledadSecrets +from leap.soledad.client.shared_db import SoledadSharedDatabase +from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.target import SoledadSyncTarget +from leap.soledad.client.sqlcipher import SQLCipherDB, SQLCipherOptions + +logger = logging.getLogger(name=__name__) + +# +# Constants +# + +SOLEDAD_CERT = None +""" +Path to the certificate file used to certify the SSL connection between +Soledad client and server. +""" + + +# +# Soledad: local encrypted storage and remote encrypted sync. +# + +class Soledad(object): + """ + Soledad provides encrypted data storage and sync. + + A Soledad instance is used to store and retrieve data in a local encrypted + database and synchronize this database with Soledad server. + + This class is also responsible for bootstrapping users' account by + creating cryptographic secrets and/or storing/fetching them on Soledad + server. + + Soledad uses ``leap.common.events`` to signal events. The possible events + to be signaled are: + + SOLEDAD_CREATING_KEYS: emitted during bootstrap sequence when key + generation starts. + SOLEDAD_DONE_CREATING_KEYS: emitted during bootstrap sequence when key + generation finishes. + SOLEDAD_UPLOADING_KEYS: emitted during bootstrap sequence when soledad + starts sending keys to server. + SOLEDAD_DONE_UPLOADING_KEYS: emitted during bootstrap sequence when + soledad finishes sending keys to server. + SOLEDAD_DOWNLOADING_KEYS: emitted during bootstrap sequence when + soledad starts to retrieve keys from server. + SOLEDAD_DONE_DOWNLOADING_KEYS: emitted during bootstrap sequence when + soledad finishes downloading keys from server. + SOLEDAD_NEW_DATA_TO_SYNC: emitted upon call to C{need_sync()} when + there's indeed new data to be synchronized between local database + replica and server's replica. + SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has + finished synchronizing with remote replica. + """ + + LOCAL_DATABASE_FILE_NAME = 'soledad.u1db' + """ + The name of the local SQLCipher U1DB database file. + """ + + STORAGE_SECRETS_FILE_NAME = "soledad.json" + """ + The name of the file where the storage secrets will be stored. + """ + + DEFAULT_PREFIX = os.path.join(get_path_prefix(), 'leap', 'soledad') + """ + Prefix for default values for path. + """ + + def __init__(self, uuid, passphrase, secrets_path, local_db_path, + server_url, cert_file, + auth_token=None, secret_id=None, defer_encryption=False): + """ + Initialize configuration, cryptographic keys and dbs. + + :param uuid: User's uuid. + :type uuid: str + + :param passphrase: The passphrase for locking and unlocking encryption + secrets for local and remote storage. + :type passphrase: unicode + + :param secrets_path: Path for storing encrypted key used for + symmetric encryption. + :type secrets_path: str + + :param local_db_path: Path for local encrypted storage db. + :type local_db_path: str + + :param server_url: URL for Soledad server. This is used either to sync + with the user's remote db and to interact with the + shared recovery database. + :type server_url: str + + :param cert_file: Path to the certificate of the ca used + to validate the SSL certificate used by the remote + soledad server. + :type cert_file: str + + :param auth_token: Authorization token for accessing remote databases. + :type auth_token: str + + :param secret_id: The id of the storage secret to be used. + :type secret_id: str + + :param defer_encryption: Whether to defer encryption/decryption of + documents, or do it inline while syncing. + :type defer_encryption: bool + + :raise BootstrapSequenceError: Raised when the secret generation and + storage on server sequence has failed + for some reason. + """ + # store config params + self._uuid = uuid + self._passphrase = passphrase + self._secrets_path = secrets_path + self._local_db_path = local_db_path + self._server_url = server_url + # configure SSL certificate + global SOLEDAD_CERT + SOLEDAD_CERT = cert_file + self._set_token(auth_token) + self._defer_encryption = defer_encryption + + self._init_config() + self._init_dirs() + + # init crypto variables + self._shared_db_instance = None + self._crypto = SoledadCrypto(self) + self._secrets = SoledadSecrets( + self._uuid, + self._passphrase, + self._secrets_path, + self._shared_db, + self._crypto, + secret_id=secret_id) + + # initiate bootstrap sequence + self._bootstrap() # might raise BootstrapSequenceError() + + def _init_config(self): + """ + Initialize configuration using default values for missing params. + """ + soledad_assert_type(self._passphrase, unicode) + # initialize secrets_path + if self._secrets_path is None: + self._secrets_path = os.path.join( + self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME) + # initialize local_db_path + if self._local_db_path is None: + self._local_db_path = os.path.join( + self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME) + # initialize server_url + soledad_assert( + self._server_url is not None, + 'Missing URL for Soledad server.') + + # + # initialization/destruction methods + # + + def _bootstrap(self): + """ + Bootstrap local Soledad instance. + + :raise BootstrapSequenceError: Raised when the secret generation and + storage on server sequence has failed for some reason. + """ + try: + self._secrets.bootstrap() + self._init_db() + except: + raise + + def _init_dirs(self): + """ + Create work directories. + + :raise OSError: in case file exists and is not a dir. + """ + paths = map( + lambda x: os.path.dirname(x), + [self._local_db_path, self._secrets_path]) + for path in paths: + try: + if not os.path.isdir(path): + logger.info('Creating directory: %s.' % path) + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + def _init_db(self): + """ + Initialize the U1DB SQLCipher database for local storage. + + Currently, Soledad uses the default SQLCipher cipher, i.e. + 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key and + uses the 'raw PRAGMA key' format to handle the key to SQLCipher. + """ + tohex = binascii.b2a_hex + # sqlcipher only accepts the hex version + key = tohex(self._secrets.get_local_storage_key()) + sync_db_key = tohex(self._secrets.get_sync_db_key()) + + opts = SQLCipherOptions( + self._local_db_path, key, + is_raw_key=True, + create=True, + defer_encryption=self._defer_encryption, + sync_db_key=sync_db_key, + crypto=self._crypto, # XXX add this + document_factory=SoledadDocument, + ) + self._db = SQLCipherDB(opts) + + def close(self): + """ + Close underlying U1DB database. + """ + logger.debug("Closing soledad") + if hasattr(self, '_db') and isinstance( + self._db, + SQLCipherDatabase): + self._db.stop_sync() + self._db.close() + + @property + def _shared_db(self): + """ + Return an instance of the shared recovery database object. + + :return: The shared database. + :rtype: SoledadSharedDatabase + """ + if self._shared_db_instance is None: + self._shared_db_instance = SoledadSharedDatabase.open_database( + urlparse.urljoin(self.server_url, SHARED_DB_NAME), + self._uuid, + False, # db should exist at this point. + creds=self._creds) + return self._shared_db_instance + + # + # Document storage, retrieval and sync. + # + + def put_doc(self, doc): + """ + Update a document in the local encrypted database. + + ============================== WARNING ============================== + This method converts the document's contents to unicode in-place. This + means that after calling C{put_doc(doc)}, the contents of the + document, i.e. C{doc.content}, might be different from before the + call. + ============================== WARNING ============================== + + :param doc: the document to update + :type doc: SoledadDocument + + :return: the new revision identifier for the document + :rtype: str + """ + doc.content = self._convert_to_unicode(doc.content) + return self._db.put_doc(doc) + + def delete_doc(self, doc): + """ + Delete a document from the local encrypted database. + + :param doc: the document to delete + :type doc: SoledadDocument + + :return: the new revision identifier for the document + :rtype: str + """ + return self._db.delete_doc(doc) + + def get_doc(self, doc_id, include_deleted=False): + """ + Retrieve a document from the local encrypted database. + + :param doc_id: the unique document identifier + :type doc_id: str + :param include_deleted: if True, deleted documents will be + returned with empty content; otherwise asking + for a deleted document will return None + :type include_deleted: bool + + :return: the document object or None + :rtype: SoledadDocument + """ + return self._db.get_doc(doc_id, include_deleted=include_deleted) + + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + """ + Get the content for many documents. + + :param doc_ids: a list of document identifiers + :type doc_ids: list + :param check_for_conflicts: if set False, then the conflict check will + be skipped, and 'None' will be returned instead of True/False + :type check_for_conflicts: bool + + :return: iterable giving the Document object for each document id + in matching doc_ids order. + :rtype: generator + """ + return self._db.get_docs( + doc_ids, check_for_conflicts=check_for_conflicts, + include_deleted=include_deleted) + + def get_all_docs(self, include_deleted=False): + """ + Get the JSON content for all documents in the database. + + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted + documents will not be included in the results. + :return: (generation, [Document]) + The current generation of the database, followed by a list of + all the documents in the database. + """ + return self._db.get_all_docs(include_deleted) + + def _convert_to_unicode(self, content): + """ + Converts content to unicode (or all the strings in content) + + NOTE: Even though this method supports any type, it will + currently ignore contents of lists, tuple or any other + iterable than dict. We don't need support for these at the + moment + + :param content: content to convert + :type content: object + + :rtype: object + """ + if isinstance(content, unicode): + return content + elif isinstance(content, str): + result = chardet.detect(content) + default = "utf-8" + encoding = result["encoding"] or default + try: + content = content.decode(encoding) + except UnicodeError as e: + logger.error("Unicode error: {0!r}. Using 'replace'".format(e)) + content = content.decode(encoding, 'replace') + return content + else: + if isinstance(content, dict): + for key in content.keys(): + content[key] = self._convert_to_unicode(content[key]) + return content + + def create_doc(self, content, doc_id=None): + """ + Create a new document in the local encrypted database. + + :param content: the contents of the new document + :type content: dict + :param doc_id: an optional identifier specifying the document id + :type doc_id: str + + :return: the new document + :rtype: SoledadDocument + """ + return self._db.create_doc( + self._convert_to_unicode(content), doc_id=doc_id) + + def create_doc_from_json(self, json, doc_id=None): + """ + Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param json: The JSON document string + :type json: str + :param doc_id: An optional identifier specifying the document id. + :type doc_id: + :return: The new document + :rtype: SoledadDocument + """ + return self._db.create_doc_from_json(json, doc_id=doc_id) + + def create_index(self, index_name, *index_expressions): + """ + Create an named index, which can then be queried for future lookups. + Creating an index which already exists is not an error, and is cheap. + Creating an index which does not match the index_expressions of the + existing index is an error. + Creating an index will block until the expressions have been evaluated + and the index generated. + + :param index_name: A unique name which can be used as a key prefix + :type index_name: str + :param index_expressions: index expressions defining the index + information. + :type index_expressions: dict + + Examples: + + "fieldname", or "fieldname.subfieldname" to index alphabetically + sorted on the contents of a field. + + "number(fieldname, width)", "lower(fieldname)" + """ + if self._db: + return self._db.create_index( + index_name, *index_expressions) + + def delete_index(self, index_name): + """ + Remove a named index. + + :param index_name: The name of the index we are removing + :type index_name: str + """ + if self._db: + return self._db.delete_index(index_name) + + def list_indexes(self): + """ + List the definitions of all known indexes. + + :return: A list of [('index-name', ['field', 'field2'])] definitions. + :rtype: list + """ + if self._db: + return self._db.list_indexes() + + def get_from_index(self, index_name, *key_values): + """ + Return documents that match the keys supplied. + + You must supply exactly the same number of values as have been defined + in the index. It is possible to do a prefix match by using '*' to + indicate a wildcard match. You can only supply '*' to trailing entries, + (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) + It is also possible to append a '*' to the last supplied value (eg + 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: List of [Document] + :rtype: list + """ + if self._db: + return self._db.get_from_index(index_name, *key_values) + + def get_count_from_index(self, index_name, *key_values): + """ + Return the count of the documents that match the keys and + values supplied. + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: count. + :rtype: int + """ + if self._db: + return self._db.get_count_from_index(index_name, *key_values) + + def get_range_from_index(self, index_name, start_value, end_value): + """ + Return documents that fall within the specified range. + + Both ends of the range are inclusive. For both start_value and + end_value, one must supply exactly the same number of values as have + been defined in the index, or pass None. In case of a single column + index, a string is accepted as an alternative for a tuple with a single + value. It is possible to do a prefix match by using '*' to indicate + a wildcard match. You can only supply '*' to trailing entries, (eg + 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also + possible to append a '*' to the last supplied value (eg 'val*', '*', + '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param start_values: tuples of values that define the lower bound of + the range. eg, if you have an index with 3 fields then you would + have: (val1, val2, val3) + :type start_values: tuple + :param end_values: tuples of values that define the upper bound of the + range. eg, if you have an index with 3 fields then you would have: + (val1, val2, val3) + :type end_values: tuple + :return: List of [Document] + :rtype: list + """ + if self._db: + return self._db.get_range_from_index( + index_name, start_value, end_value) + + def get_index_keys(self, index_name): + """ + Return all keys under which documents are indexed in this index. + + :param index_name: The index to query + :type index_name: str + :return: [] A list of tuples of indexed keys. + :rtype: list + """ + if self._db: + return self._db.get_index_keys(index_name) + + def get_doc_conflicts(self, doc_id): + """ + Get the list of conflicts for the given document. + + :param doc_id: the document id + :type doc_id: str + + :return: a list of the document entries that are conflicted + :rtype: list + """ + if self._db: + return self._db.get_doc_conflicts(doc_id) + + def resolve_doc(self, doc, conflicted_doc_revs): + """ + Mark a document as no longer conflicted. + + :param doc: a document with the new content to be inserted. + :type doc: SoledadDocument + :param conflicted_doc_revs: a list of revisions that the new content + supersedes. + :type conflicted_doc_revs: list + """ + if self._db: + return self._db.resolve_doc(doc, conflicted_doc_revs) + + def sync(self, defer_decryption=True): + """ + Synchronize the local encrypted replica with a remote replica. + + This method blocks until a syncing lock is acquired, so there are no + attempts of concurrent syncs from the same client replica. + + :param url: the url of the target replica to sync with + :type url: str + + :param defer_decryption: Whether to defer the decryption process using + the intermediate database. If False, + decryption will be done inline. + :type defer_decryption: bool + + :return: The local generation before the synchronisation was + performed. + :rtype: str + """ + if self._db: + try: + local_gen = self._db.sync( + urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), + creds=self._creds, autocreate=False, + defer_decryption=defer_decryption) + soledad_events.signal( + soledad_events.SOLEDAD_DONE_DATA_SYNC, self._uuid) + return local_gen + except Exception as e: + logger.error("Soledad exception when syncing: %s" % str(e)) + + def stop_sync(self): + """ + Stop the current syncing process. + """ + if self._db: + self._db.stop_sync() + + def need_sync(self, url): + """ + Return if local db replica differs from remote url's replica. + + :param url: The remote replica to compare with local replica. + :type url: str + + :return: Whether remote replica and local replica differ. + :rtype: bool + """ + target = SoledadSyncTarget( + url, self._db._get_replica_uid(), creds=self._creds, + crypto=self._crypto) + info = target.get_sync_info(self._db._get_replica_uid()) + # compare source generation with target's last known source generation + if self._db._get_generation() != info[4]: + soledad_events.signal( + soledad_events.SOLEDAD_NEW_DATA_TO_SYNC, self._uuid) + return True + return False + + @property + def syncing(self): + """ + Property, True if the syncer is syncing. + """ + return self._db.syncing + + def _set_token(self, token): + """ + Set the authentication token for remote database access. + + Build the credentials dictionary with the following format: + + self._{ + 'token': { + 'uuid': '' + 'token': '' + } + + :param token: The authentication token. + :type token: str + """ + self._creds = { + 'token': { + 'uuid': self._uuid, + 'token': token, + } + } + + def _get_token(self): + """ + Return current token from credentials dictionary. + """ + return self._creds['token']['token'] + + token = property(_get_token, _set_token, doc='The authentication Token.') + + # + # Setters/getters + # + + def _get_uuid(self): + return self._uuid + + uuid = property(_get_uuid, doc='The user uuid.') + + def get_secret_id(self): + return self._secrets.secret_id + + def set_secret_id(self, secret_id): + self._secrets.set_secret_id(secret_id) + + secret_id = property( + get_secret_id, + set_secret_id, + doc='The active secret id.') + + def _set_secrets_path(self, secrets_path): + self._secrets.secrets_path = secrets_path + + def _get_secrets_path(self): + return self._secrets.secrets_path + + secrets_path = property( + _get_secrets_path, + _set_secrets_path, + doc='The path for the file containing the encrypted symmetric secret.') + + def _get_local_db_path(self): + return self._local_db_path + + local_db_path = property( + _get_local_db_path, + doc='The path for the local database replica.') + + def _get_server_url(self): + return self._server_url + + server_url = property( + _get_server_url, + doc='The URL of the Soledad server.') + + @property + def storage_secret(self): + """ + Return the secret used for symmetric encryption. + """ + return self._secrets.storage_secret + + @property + def remote_storage_secret(self): + """ + Return the secret used for encryption of remotely stored data. + """ + return self._secrets.remote_storage_secret + + @property + def secrets(self): + return self._secrets + + @property + def passphrase(self): + return self._secrets.passphrase + + def change_passphrase(self, new_passphrase): + """ + Change the passphrase that encrypts the storage secret. + + :param new_passphrase: The new passphrase. + :type new_passphrase: unicode + + :raise NoStorageSecret: Raised if there's no storage secret available. + """ + self._secrets.change_passphrase(new_passphrase) + + +# ---------------------------------------------------------------------------- +# Monkey patching u1db to be able to provide a custom SSL cert +# ---------------------------------------------------------------------------- + +# We need a more reasonable timeout (in seconds) +SOLEDAD_TIMEOUT = 120 + + +class VerifiedHTTPSConnection(httplib.HTTPSConnection): + """ + HTTPSConnection verifying server side certificates. + """ + # derived from httplib.py + + def connect(self): + """ + Connect to a host on a given (SSL) port. + """ + try: + source = self.source_address + sock = socket.create_connection((self.host, self.port), + SOLEDAD_TIMEOUT, source) + except AttributeError: + # source_address was introduced in 2.7 + sock = socket.create_connection((self.host, self.port), + SOLEDAD_TIMEOUT) + if self._tunnel_host: + self.sock = sock + self._tunnel() + + self.sock = ssl.wrap_socket(sock, + ca_certs=SOLEDAD_CERT, + cert_reqs=ssl.CERT_REQUIRED) + match_hostname(self.sock.getpeercert(), self.host) + + +old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection +http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection + diff --git a/client/src/leap/soledad/client/mp_safe_db.py b/client/src/leap/soledad/client/mp_safe_db.py deleted file mode 100644 index 9ed0bef4..00000000 --- a/client/src/leap/soledad/client/mp_safe_db.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -# mp_safe_db.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Multiprocessing-safe SQLite database. -""" - - -from threading import Thread -from Queue import Queue -from pysqlcipher import dbapi2 - - -# Thanks to http://code.activestate.com/recipes/526618/ - -class MPSafeSQLiteDB(Thread): - """ - A multiprocessing-safe SQLite database accessor. - """ - - CLOSE = "--close--" - NO_MORE = "--no more--" - - def __init__(self, db_path): - """ - Initialize the process - """ - Thread.__init__(self) - self._db_path = db_path - self._requests = Queue() - self.start() - - def run(self): - """ - Run the multiprocessing-safe database accessor. - """ - conn = dbapi2.connect(self._db_path) - while True: - req, arg, res = self._requests.get() - if req == self.CLOSE: - break - with conn: - cursor = conn.cursor() - cursor.execute(req, arg) - if res: - for rec in cursor.fetchall(): - res.put(rec) - res.put(self.NO_MORE) - conn.close() - - def execute(self, req, arg=None, res=None): - """ - Execute a request on the database. - - :param req: The request to be executed. - :type req: str - :param arg: The arguments for the request. - :type arg: tuple - :param res: A queue to write request results. - :type res: multiprocessing.Queue - """ - self._requests.put((req, arg or tuple(), res)) - - def select(self, req, arg=None): - """ - Run a select query on the database and yield results. - - :param req: The request to be executed. - :type req: str - :param arg: The arguments for the request. - :type arg: tuple - """ - res = Queue() - self.execute(req, arg, res) - while True: - rec = res.get() - if rec == self.NO_MORE: - break - yield rec - - def close(self): - """ - Close the database connection. - """ - self.execute(self.CLOSE) - self.join() - - def cursor(self): - """ - Return a fake cursor object. - - Not really a cursor, but allows for calling db.cursor().execute(). - - :return: Self. - :rtype: MPSafeSQLiteDatabase - """ - return self diff --git a/client/src/leap/soledad/client/mp_safe_db_TOREMOVE.py b/client/src/leap/soledad/client/mp_safe_db_TOREMOVE.py new file mode 100644 index 00000000..9ed0bef4 --- /dev/null +++ b/client/src/leap/soledad/client/mp_safe_db_TOREMOVE.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# mp_safe_db.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Multiprocessing-safe SQLite database. +""" + + +from threading import Thread +from Queue import Queue +from pysqlcipher import dbapi2 + + +# Thanks to http://code.activestate.com/recipes/526618/ + +class MPSafeSQLiteDB(Thread): + """ + A multiprocessing-safe SQLite database accessor. + """ + + CLOSE = "--close--" + NO_MORE = "--no more--" + + def __init__(self, db_path): + """ + Initialize the process + """ + Thread.__init__(self) + self._db_path = db_path + self._requests = Queue() + self.start() + + def run(self): + """ + Run the multiprocessing-safe database accessor. + """ + conn = dbapi2.connect(self._db_path) + while True: + req, arg, res = self._requests.get() + if req == self.CLOSE: + break + with conn: + cursor = conn.cursor() + cursor.execute(req, arg) + if res: + for rec in cursor.fetchall(): + res.put(rec) + res.put(self.NO_MORE) + conn.close() + + def execute(self, req, arg=None, res=None): + """ + Execute a request on the database. + + :param req: The request to be executed. + :type req: str + :param arg: The arguments for the request. + :type arg: tuple + :param res: A queue to write request results. + :type res: multiprocessing.Queue + """ + self._requests.put((req, arg or tuple(), res)) + + def select(self, req, arg=None): + """ + Run a select query on the database and yield results. + + :param req: The request to be executed. + :type req: str + :param arg: The arguments for the request. + :type arg: tuple + """ + res = Queue() + self.execute(req, arg, res) + while True: + rec = res.get() + if rec == self.NO_MORE: + break + yield rec + + def close(self): + """ + Close the database connection. + """ + self.execute(self.CLOSE) + self.join() + + def cursor(self): + """ + Return a fake cursor object. + + Not really a cursor, but allows for calling db.cursor().execute(). + + :return: Self. + :rtype: MPSafeSQLiteDatabase + """ + return self diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 613903f7..fcef592d 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -53,7 +53,7 @@ from contextlib import contextmanager from collections import defaultdict from httplib import CannotSendRequest -from pysqlcipher import dbapi2 +from pysqlcipher import dbapi2 as sqlcipher_dbapi2 from u1db.backends import sqlite_backend from u1db import errors as u1db_errors from taskthread import TimerTask @@ -71,7 +71,7 @@ from leap.soledad.common.document import SoledadDocument logger = logging.getLogger(__name__) # Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2 -sqlite_backend.dbapi2 = dbapi2 +sqlite_backend.dbapi2 = sqlcipher_dbapi2 # It seems that, as long as we are not using old sqlite versions, serialized # mode is enabled by default at compile time. So accessing db connections from @@ -91,11 +91,12 @@ SQLITE_CHECK_SAME_THREAD = False SQLITE_ISOLATION_LEVEL = None +# TODO accept cyrpto object too.... or pass it along.. class SQLCipherOptions(object): def __init__(self, path, key, create=True, is_raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, - document_factory=None, defer_encryption=False, - sync_db_key=None): + document_factory=None, + defer_encryption=False, sync_db_key=None): """ Options for the initialization of an SQLCipher database. @@ -140,39 +141,39 @@ class SQLCipherOptions(object): # XXX Use SQLCIpherOptions instead -def open(path, password, create=True, document_factory=None, crypto=None, - raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, - cipher_page_size=1024, defer_encryption=False, sync_db_key=None): - """ - Open a database at the given location. - - *** IMPORTANT *** - - Don't forget to close the database after use by calling the close() - method otherwise some resources might not be freed and you may experience - several kinds of leakages. - - *** IMPORTANT *** - - Will raise u1db.errors.DatabaseDoesNotExist if create=False and the - database does not already exist. - - :return: An instance of Database. - :rtype SQLCipherDatabase - """ - args = (path, password) - kwargs = { - 'create': create, - 'document_factory': document_factory, - 'crypto': crypto, - 'raw_key': raw_key, - 'cipher': cipher, - 'kdf_iter': kdf_iter, - 'cipher_page_size': cipher_page_size, - 'defer_encryption': defer_encryption, - 'sync_db_key': sync_db_key} +#def open(path, password, create=True, document_factory=None, crypto=None, + #raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, + #cipher_page_size=1024, defer_encryption=False, sync_db_key=None): + #""" + #Open a database at the given location. +# + #*** IMPORTANT *** +# + #Don't forget to close the database after use by calling the close() + #method otherwise some resources might not be freed and you may experience + #several kinds of leakages. +# + #*** IMPORTANT *** +# + #Will raise u1db.errors.DatabaseDoesNotExist if create=False and the + #database does not already exist. +# + #:return: An instance of Database. + #:rtype SQLCipherDatabase + #""" + #args = (path, password) + #kwargs = { + #'create': create, + #'document_factory': document_factory, + #'crypto': crypto, + #'raw_key': raw_key, + #'cipher': cipher, + #'kdf_iter': kdf_iter, + #'cipher_page_size': cipher_page_size, + #'defer_encryption': defer_encryption, + #'sync_db_key': sync_db_key} # XXX pass only a CryptoOptions object around - return SQLCipherDatabase.open_database(*args, **kwargs) + #return SQLCipherDatabase.open_database(*args, **kwargs) # @@ -216,9 +217,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ # XXX Use SQLCIpherOptions instead - def __init__(self, sqlcipher_file, password, document_factory=None, - crypto=None, raw_key=False, cipher='aes-256-cbc', - kdf_iter=4000, cipher_page_size=1024, sync_db_key=None): + def __init__(self, opts): """ Connect to an existing SQLCipher database, creating a new sqlcipher database file if needed. @@ -230,23 +229,28 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): experience several kinds of leakages. *** IMPORTANT *** + + :param opts: + :type opts: SQLCipherOptions """ # ensure the db is encrypted if the file already exists - if os.path.exists(sqlcipher_file): - # XXX pass only a CryptoOptions object around - self.assert_db_is_encrypted( - sqlcipher_file, password, raw_key, cipher, kdf_iter, - cipher_page_size) + if os.path.exists(opts.sqlcipher_file): + self.assert_db_is_encrypted(opts) # connect to the sqlcipher database + # XXX this lock should not be needed ----------------- + # u1db holds a mutex over sqlite internally for the initialization. with self.k_lock: - self._db_handle = dbapi2.connect( - sqlcipher_file, + self._db_handle = sqlcipher_dbapi2.connect( + + # TODO ----------------------------------------------- + # move the init to a single function + opts.sqlcipher_file, isolation_level=SQLITE_ISOLATION_LEVEL, check_same_thread=SQLITE_CHECK_SAME_THREAD) # set SQLCipher cryptographic parameters - # XXX allow optiona deferredChain here ? + # XXX allow optional deferredChain here ? pragmas.set_crypto_pragmas( self._db_handle, password, raw_key, cipher, kdf_iter, cipher_page_size) @@ -260,8 +264,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self._real_replica_uid = None self._ensure_schema() - self._crypto = crypto + self._crypto = opts.crypto + + # TODO ------------------------------------------------ + # Move syncdb to another class ------------------------ # define sync-db attrs self._sqlcipher_file = sqlcipher_file self._sync_db_key = sync_db_key @@ -294,103 +301,122 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # self._syncers = {'': ('', syncer), ...} self._syncers = {} - @classmethod - # XXX Use SQLCIpherOptions instead - def _open_database(cls, sqlcipher_file, password, document_factory=None, - crypto=None, raw_key=False, cipher='aes-256-cbc', - kdf_iter=4000, cipher_page_size=1024, - defer_encryption=False, sync_db_key=None): + def _extra_schema_init(self, c): """ - Open a SQLCipher database. + Add any extra fields, etc to the basic table definitions. - :return: The database object. - :rtype: SQLCipherDatabase + This method is called by u1db.backends.sqlite_backend._initialize() + method, which is executed when the database schema is created. Here, + we use it to include the "syncable" property for LeapDocuments. + + :param c: The cursor for querying the database. + :type c: dbapi2.cursor """ - cls.defer_encryption = defer_encryption - if not os.path.isfile(sqlcipher_file): - raise u1db_errors.DatabaseDoesNotExist() + c.execute( + 'ALTER TABLE document ' + 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') + - tries = 2 + # TODO ---- rescue the fix for the windows case from here... + #@classmethod + # XXX Use SQLCIpherOptions instead + #def _open_database(cls, sqlcipher_file, password, document_factory=None, + #crypto=None, raw_key=False, cipher='aes-256-cbc', + #kdf_iter=4000, cipher_page_size=1024, + #defer_encryption=False, sync_db_key=None): + #""" + #Open a SQLCipher database. +# + #:return: The database object. + #:rtype: SQLCipherDatabase + #""" + #cls.defer_encryption = defer_encryption + #if not os.path.isfile(sqlcipher_file): + #raise u1db_errors.DatabaseDoesNotExist() +# + #tries = 2 # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6) # where without re-opening the database on Windows, it # doesn't see the transaction that was just committed - while True: - - with cls.k_lock: - db_handle = dbapi2.connect( - sqlcipher_file, - check_same_thread=SQLITE_CHECK_SAME_THREAD) - - try: + #while True: +# + #with cls.k_lock: + #db_handle = dbapi2.connect( + #sqlcipher_file, + #check_same_thread=SQLITE_CHECK_SAME_THREAD) +# + #try: # set cryptographic params - +# # XXX pass only a CryptoOptions object around - pragmas.set_crypto_pragmas( - db_handle, password, raw_key, cipher, kdf_iter, - cipher_page_size) - c = db_handle.cursor() + #pragmas.set_crypto_pragmas( + #db_handle, password, raw_key, cipher, kdf_iter, + #cipher_page_size) + #c = db_handle.cursor() # XXX if we use it here, it should be public - v, err = cls._which_index_storage(c) - except Exception as exc: - logger.warning("ERROR OPENING DATABASE!") - logger.debug("error was: %r" % exc) - v, err = None, exc - finally: - db_handle.close() - if v is not None: - break + #v, err = cls._which_index_storage(c) + #except Exception as exc: + #logger.warning("ERROR OPENING DATABASE!") + #logger.debug("error was: %r" % exc) + #v, err = None, exc + #finally: + #db_handle.close() + #if v is not None: + #break # possibly another process is initializing it, wait for it to be # done - if tries == 0: - raise err # go for the richest error? - tries -= 1 - time.sleep(cls.WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL) - return SQLCipherDatabase._sqlite_registry[v]( - sqlcipher_file, password, document_factory=document_factory, - crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, - cipher_page_size=cipher_page_size, sync_db_key=sync_db_key) - - @classmethod - def open_database(cls, sqlcipher_file, password, create, - document_factory=None, crypto=None, raw_key=False, - cipher='aes-256-cbc', kdf_iter=4000, - cipher_page_size=1024, defer_encryption=False, - sync_db_key=None): + #if tries == 0: + #raise err # go for the richest error? + #tries -= 1 + #time.sleep(cls.WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL) + #return SQLCipherDatabase._sqlite_registry[v]( + #sqlcipher_file, password, document_factory=document_factory, + #crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, + #cipher_page_size=cipher_page_size, sync_db_key=sync_db_key) + + #@classmethod + #def open_database(cls, sqlcipher_file, password, create, + #document_factory=None, crypto=None, raw_key=False, + #cipher='aes-256-cbc', kdf_iter=4000, + #cipher_page_size=1024, defer_encryption=False, + #sync_db_key=None): # XXX pass only a CryptoOptions object around - """ - Open a SQLCipher database. - - *** IMPORTANT *** - - Don't forget to close the database after use by calling the close() - method otherwise some resources might not be freed and you may - experience several kinds of leakages. - - *** IMPORTANT *** - - :return: The database object. - :rtype: SQLCipherDatabase - """ - cls.defer_encryption = defer_encryption - args = sqlcipher_file, password - kwargs = { - 'crypto': crypto, - 'raw_key': raw_key, - 'cipher': cipher, - 'kdf_iter': kdf_iter, - 'cipher_page_size': cipher_page_size, - 'defer_encryption': defer_encryption, - 'sync_db_key': sync_db_key, - 'document_factory': document_factory, - } - try: - return cls._open_database(*args, **kwargs) - except u1db_errors.DatabaseDoesNotExist: - if not create: - raise - + #""" + #Open a SQLCipher database. +# + #*** IMPORTANT *** +# + #Don't forget to close the database after use by calling the close() + #method otherwise some resources might not be freed and you may + #experience several kinds of leakages. +# + #*** IMPORTANT *** +# + #:return: The database object. + #:rtype: SQLCipherDatabase + #""" + #cls.defer_encryption = defer_encryption + #args = sqlcipher_file, password + #kwargs = { + #'crypto': crypto, + #'raw_key': raw_key, + #'cipher': cipher, + #'kdf_iter': kdf_iter, + #'cipher_page_size': cipher_page_size, + #'defer_encryption': defer_encryption, + #'sync_db_key': sync_db_key, + #'document_factory': document_factory, + #} + #try: + #return cls._open_database(*args, **kwargs) + #except u1db_errors.DatabaseDoesNotExist: + #if not create: + #raise +# # XXX here we were missing sync_db_key, intentional? - return SQLCipherDatabase(*args, **kwargs) + #return SQLCipherDatabase(*args, **kwargs) + + # BEGIN SYNC FOO ---------------------------------------------------------- def sync(self, url, creds=None, autocreate=True, defer_decryption=True): """ @@ -471,7 +497,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): def _get_syncer(self, url, creds=None): """ - Get a synchronizer for C{url} using C{creds}. + Get a synchronizer for ``url`` using ``creds``. :param url: The url of the target replica to sync with. :type url: str @@ -504,20 +530,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): syncer.num_inserted = 0 return syncer - def _extra_schema_init(self, c): - """ - Add any extra fields, etc to the basic table definitions. - - This method is called by u1db.backends.sqlite_backend._initialize() - method, which is executed when the database schema is created. Here, - we use it to include the "syncable" property for LeapDocuments. - - :param c: The cursor for querying the database. - :type c: dbapi2.cursor - """ - c.execute( - 'ALTER TABLE document ' - 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') + # END SYNC FOO ---------------------------------------------------------- def _init_sync_db(self): """ @@ -601,8 +614,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :return: The new document revision. :rtype: str """ - doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc( - self, doc) + doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc) if self.defer_encryption: self.sync_queue.put_nowait(doc) return doc_rev @@ -644,8 +656,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self, doc_id, check_for_conflicts) if doc: c = self._db_handle.cursor() - c.execute('SELECT syncable FROM document ' - 'WHERE doc_id=?', + c.execute('SELECT syncable FROM document WHERE doc_id=?', (doc.doc_id,)) result = c.fetchone() doc.syncable = bool(result[0]) @@ -691,11 +702,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # backend should raise a DatabaseError exception. sqlite_backend.SQLitePartialExpandDatabase(sqlcipher_file) raise DatabaseIsNotEncrypted() - except dbapi2.DatabaseError: + except sqlcipher_dbapi2.DatabaseError: # assert that we can access it using SQLCipher with the given # key with cls.k_lock: - db_handle = dbapi2.connect( + db_handle = sqlcipher_dbapi2.connect( sqlcipher_file, isolation_level=SQLITE_ISOLATION_LEVEL, check_same_thread=SQLITE_CHECK_SAME_THREAD) @@ -750,8 +761,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): )) try: c.execute(statement, tuple(args)) - except dbapi2.OperationalError, e: - raise dbapi2.OperationalError( + except sqlcipher_dbapi2.OperationalError, e: + raise sqlcipher_dbapi2.OperationalError( str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) res = c.fetchall() return res[0][0] @@ -760,6 +771,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ Close db_handle and close syncer. """ + # TODO separate db from syncers -------------- + if logger is not None: # logger might be none if called from __del__ logger.debug("Sqlcipher backend: closing") # stop the sync watcher for deferred encryption @@ -780,6 +793,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): if self._db_handle is not None: self._db_handle.close() self._db_handle = None + + # --------------------------------------- # close the sync database if self._sync_db is not None: self._sync_db.close() diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index 0297c75c..a47afbb6 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -14,8 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ Soledad synchronization utilities. @@ -27,8 +25,6 @@ Extend u1db Synchronizer with the ability to: * Be interrupted and recovered. """ - - import logging import traceback from threading import Lock -- cgit v1.2.3 From e0f70a342deccbb53a6ea7215b3322388bb18461 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 23 Sep 2014 13:38:06 -0500 Subject: Refactor soledad api to use async db * add examples and benchmarks * remove autocommit mode, allow wal disabling * lock initialization * make api use async calls --- .gitignore | 1 + client/src/leap/soledad/client/adbapi.py | 146 +++- client/src/leap/soledad/client/api.py | 323 ++++---- client/src/leap/soledad/client/examples/README | 4 + .../src/leap/soledad/client/examples/compare.txt | 8 + .../src/leap/soledad/client/examples/manifest.phk | 50 ++ .../leap/soledad/client/examples/plot-async-db.py | 45 ++ .../leap/soledad/client/examples/run_benchmark.py | 28 + .../src/leap/soledad/client/examples/use_adbapi.py | 103 +++ client/src/leap/soledad/client/examples/use_api.py | 67 ++ .../src/leap/soledad/client/mp_safe_db_TOREMOVE.py | 112 --- client/src/leap/soledad/client/pragmas.py | 20 +- client/src/leap/soledad/client/sqlcipher.py | 845 ++++++++++----------- 13 files changed, 991 insertions(+), 761 deletions(-) create mode 100644 client/src/leap/soledad/client/examples/README create mode 100644 client/src/leap/soledad/client/examples/compare.txt create mode 100644 client/src/leap/soledad/client/examples/manifest.phk create mode 100644 client/src/leap/soledad/client/examples/plot-async-db.py create mode 100644 client/src/leap/soledad/client/examples/run_benchmark.py create mode 100644 client/src/leap/soledad/client/examples/use_adbapi.py create mode 100644 client/src/leap/soledad/client/examples/use_api.py delete mode 100644 client/src/leap/soledad/client/mp_safe_db_TOREMOVE.py diff --git a/.gitignore b/.gitignore index bd170f79..c502541e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ MANIFEST *.pyc *.log *.*~ +*.csv diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index 730999a3..3b15509b 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# sqlcipher.py +# adbapi.py # Copyright (C) 2013, 2014 LEAP # # This program is free software: you can redistribute it and/or modify @@ -17,61 +17,135 @@ """ An asyncrhonous interface to soledad using sqlcipher backend. It uses twisted.enterprise.adbapi. - """ +import re import os import sys +from functools import partial + +import u1db +from u1db.backends import sqlite_backend + from twisted.enterprise import adbapi from twisted.python import log +from leap.soledad.client.sqlcipher import set_init_pragmas + + DEBUG_SQL = os.environ.get("LEAP_DEBUG_SQL") if DEBUG_SQL: log.startLogging(sys.stdout) -def getConnectionPool(db=None, key=None): - return SQLCipherConnectionPool( - "pysqlcipher.dbapi2", database=db, key=key, check_same_thread=False) +def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): + if openfun is None and driver == "pysqlcipher": + openfun = partial(set_init_pragmas, opts=opts) + return U1DBConnectionPool( + "%s.dbapi2" % driver, database=opts.path, + check_same_thread=False, cp_openfun=openfun) -class SQLCipherConnectionPool(adbapi.ConnectionPool): +# XXX work in progress -------------------------------------------- - key = None - def connect(self): - """ - Return a database connection when one becomes available. +class U1DBSqliteWrapper(sqlite_backend.SQLitePartialExpandDatabase): + """ + A very simple wrapper around sqlcipher backend. - This method blocks and should be run in a thread from the internal - threadpool. Don't call this method directly from non-threaded code. - Using this method outside the external threadpool may exceed the - maximum number of connections in the pool. + Instead of initializing the database on the fly, it just uses an existing + connection that is passed to it in the initializer. + """ - :return: a database connection from the pool. - """ - self.noisy = DEBUG_SQL + def __init__(self, conn): + self._db_handle = conn + self._real_replica_uid = None + self._ensure_schema() + self._factory = u1db.Document - tid = self.threadID() - conn = self.connections.get(tid) - if self.key is None: - self.key = self.connkw.pop('key', None) +class U1DBConnection(adbapi.Connection): - if conn is None: - if self.noisy: - log.msg('adbapi connecting: %s %s%s' % (self.dbapiName, - self.connargs or '', - self.connkw or '')) - conn = self.dbapi.connect(*self.connargs, **self.connkw) + u1db_wrapper = U1DBSqliteWrapper + + def __init__(self, pool, init_u1db=False): + self.init_u1db = init_u1db + adbapi.Connection.__init__(self, pool) + + def reconnect(self): + if self._connection is not None: + self._pool.disconnect(self._connection) + self._connection = self._pool.connect() + + if self.init_u1db: + self._u1db = self.u1db_wrapper(self._connection) + + def __getattr__(self, name): + if name.startswith('u1db_'): + meth = re.sub('^u1db_', '', name) + return getattr(self._u1db, meth) + else: + return getattr(self._connection, name) - # XXX we should hook here all OUR SOLEDAD pragmas ----- - conn.cursor().execute("PRAGMA key=%s" % self.key) - conn.commit() - # ----------------------------------------------------- - # XXX profit of openfun isntead??? - if self.openfun is not None: - self.openfun(conn) - self.connections[tid] = conn - return conn +class U1DBTransaction(adbapi.Transaction): + + def __getattr__(self, name): + if name.startswith('u1db_'): + meth = re.sub('^u1db_', '', name) + return getattr(self._connection._u1db, meth) + else: + return getattr(self._cursor, name) + + +class U1DBConnectionPool(adbapi.ConnectionPool): + + connectionFactory = U1DBConnection + transactionFactory = U1DBTransaction + + def __init__(self, *args, **kwargs): + adbapi.ConnectionPool.__init__(self, *args, **kwargs) + # all u1db connections, hashed by thread-id + self.u1dbconnections = {} + + def runU1DBQuery(self, meth, *args, **kw): + meth = "u1db_%s" % meth + return self.runInteraction(self._runU1DBQuery, meth, *args, **kw) + + def _runU1DBQuery(self, trans, meth, *args, **kw): + meth = getattr(trans, meth) + return meth(*args, **kw) + + def _runInteraction(self, interaction, *args, **kw): + tid = self.threadID() + u1db = self.u1dbconnections.get(tid) + conn = self.connectionFactory(self, init_u1db=not bool(u1db)) + + if u1db is None: + self.u1dbconnections[tid] = conn._u1db + else: + conn._u1db = u1db + + trans = self.transactionFactory(self, conn) + try: + result = interaction(trans, *args, **kw) + trans.close() + conn.commit() + return result + except: + excType, excValue, excTraceback = sys.exc_info() + try: + conn.rollback() + except: + log.err(None, "Rollback failed") + raise excType, excValue, excTraceback + + def finalClose(self): + self.shutdownID = None + self.threadpool.stop() + self.running = False + for conn in self.connections.values(): + self._close(conn) + for u1db in self.u1dbconnections.values(): + self._close(u1db) + self.connections.clear() diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index bfb6c703..703b9516 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -34,7 +34,6 @@ import socket import ssl import urlparse - try: import cchardet as chardet except ImportError: @@ -47,15 +46,14 @@ from leap.common.config import get_path_prefix from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type -from leap.soledad.common.document import SoledadDocument +from leap.soledad.client import adbapi from leap.soledad.client import events as soledad_events from leap.soledad.client.crypto import SoledadCrypto from leap.soledad.client.secrets import SoledadSecrets from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client.sqlcipher import SQLCipherDatabase from leap.soledad.client.target import SoledadSyncTarget -from leap.soledad.client.sqlcipher import SQLCipherDB, SQLCipherOptions +from leap.soledad.client.sqlcipher import SQLCipherOptions logger = logging.getLogger(name=__name__) @@ -200,18 +198,19 @@ class Soledad(object): Initialize configuration using default values for missing params. """ soledad_assert_type(self._passphrase, unicode) + initialize = lambda attr, val: attr is None and setattr(attr, val) + # initialize secrets_path - if self._secrets_path is None: - self._secrets_path = os.path.join( - self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME) + initialize(self._secrets_path, os.path.join( + self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME)) + # initialize local_db_path - if self._local_db_path is None: - self._local_db_path = os.path.join( - self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME) + initialize(self._local_db_path, os.path.join( + self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME)) + # initialize server_url - soledad_assert( - self._server_url is not None, - 'Missing URL for Soledad server.') + soledad_assert(self._server_url is not None, + 'Missing URL for Soledad server.') # # initialization/destruction methods @@ -221,14 +220,13 @@ class Soledad(object): """ Bootstrap local Soledad instance. - :raise BootstrapSequenceError: Raised when the secret generation and - storage on server sequence has failed for some reason. + :raise BootstrapSequenceError: + Raised when the secret generation and storage on server sequence + has failed for some reason. """ - try: - self._secrets.bootstrap() - self._init_db() - except: - raise + self._secrets.bootstrap() + self._init_db() + # XXX initialize syncers? def _init_dirs(self): """ @@ -255,8 +253,9 @@ class Soledad(object): Initialize the U1DB SQLCipher database for local storage. Currently, Soledad uses the default SQLCipher cipher, i.e. - 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key and - uses the 'raw PRAGMA key' format to handle the key to SQLCipher. + 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key, + and internally the SQLCipherDatabase initialization uses the 'raw + PRAGMA key' format to handle the key to SQLCipher. """ tohex = binascii.b2a_hex # sqlcipher only accepts the hex version @@ -265,25 +264,28 @@ class Soledad(object): opts = SQLCipherOptions( self._local_db_path, key, - is_raw_key=True, - create=True, + is_raw_key=True, create=True, defer_encryption=self._defer_encryption, sync_db_key=sync_db_key, - crypto=self._crypto, # XXX add this - document_factory=SoledadDocument, ) - self._db = SQLCipherDB(opts) + self._dbpool = adbapi.getConnectionPool(opts) def close(self): """ Close underlying U1DB database. """ logger.debug("Closing soledad") - if hasattr(self, '_db') and isinstance( - self._db, - SQLCipherDatabase): - self._db.stop_sync() - self._db.close() + self._dbpool.close() + + # TODO close syncers >>>>>> + + #if hasattr(self, '_db') and isinstance( + #self._db, + #SQLCipherDatabase): + #self._db.close() +# + # XXX stop syncers + # self._db.stop_sync() @property def _shared_db(self): @@ -306,24 +308,29 @@ class Soledad(object): # def put_doc(self, doc): + # TODO what happens with this warning during the deferred life cycle? + # Isn't it better to defend ourselves from the mutability, to avoid + # nasty surprises? """ Update a document in the local encrypted database. ============================== WARNING ============================== This method converts the document's contents to unicode in-place. This - means that after calling C{put_doc(doc)}, the contents of the - document, i.e. C{doc.content}, might be different from before the + means that after calling `put_doc(doc)`, the contents of the + document, i.e. `doc.content`, might be different from before the call. ============================== WARNING ============================== :param doc: the document to update :type doc: SoledadDocument - :return: the new revision identifier for the document - :rtype: str + :return: + a deferred that will fire with the new revision identifier for + the document + :rtype: Deferred """ doc.content = self._convert_to_unicode(doc.content) - return self._db.put_doc(doc) + return self._dbpool.put_doc(doc) def delete_doc(self, doc): """ @@ -332,10 +339,12 @@ class Soledad(object): :param doc: the document to delete :type doc: SoledadDocument - :return: the new revision identifier for the document - :rtype: str + :return: + a deferred that will fire with ... + :rtype: Deferred """ - return self._db.delete_doc(doc) + # XXX what does this do when fired??? + return self._dbpool.delete_doc(doc) def get_doc(self, doc_id, include_deleted=False): """ @@ -343,15 +352,17 @@ class Soledad(object): :param doc_id: the unique document identifier :type doc_id: str - :param include_deleted: if True, deleted documents will be - returned with empty content; otherwise asking - for a deleted document will return None + :param include_deleted: + if True, deleted documents will be returned with empty content; + otherwise asking for a deleted document will return None :type include_deleted: bool - :return: the document object or None - :rtype: SoledadDocument + :return: + A deferred that will fire with the document object, containing a + SoledadDocument, or None if it could not be found + :rtype: Deferred """ - return self._db.get_doc(doc_id, include_deleted=include_deleted) + return self._dbpool.get_doc(doc_id, include_deleted=include_deleted) def get_docs(self, doc_ids, check_for_conflicts=True, include_deleted=False): @@ -364,11 +375,12 @@ class Soledad(object): be skipped, and 'None' will be returned instead of True/False :type check_for_conflicts: bool - :return: iterable giving the Document object for each document id - in matching doc_ids order. - :rtype: generator + :return: + A deferred that will fire with an iterable giving the Document + object for each document id in matching doc_ids order. + :rtype: Deferred """ - return self._db.get_docs( + return self._dbpool.get_docs( doc_ids, check_for_conflicts=check_for_conflicts, include_deleted=include_deleted) @@ -379,43 +391,13 @@ class Soledad(object): :param include_deleted: If set to True, deleted documents will be returned with empty content. Otherwise deleted documents will not be included in the results. - :return: (generation, [Document]) - The current generation of the database, followed by a list of - all the documents in the database. + :return: + A deferred that will fire with (generation, [Document]): that is, + the current generation of the database, followed by a list of all + the documents in the database. + :rtype: Deferred """ - return self._db.get_all_docs(include_deleted) - - def _convert_to_unicode(self, content): - """ - Converts content to unicode (or all the strings in content) - - NOTE: Even though this method supports any type, it will - currently ignore contents of lists, tuple or any other - iterable than dict. We don't need support for these at the - moment - - :param content: content to convert - :type content: object - - :rtype: object - """ - if isinstance(content, unicode): - return content - elif isinstance(content, str): - result = chardet.detect(content) - default = "utf-8" - encoding = result["encoding"] or default - try: - content = content.decode(encoding) - except UnicodeError as e: - logger.error("Unicode error: {0!r}. Using 'replace'".format(e)) - content = content.decode(encoding, 'replace') - return content - else: - if isinstance(content, dict): - for key in content.keys(): - content[key] = self._convert_to_unicode(content[key]) - return content + return self._dbpool.get_all_docs(include_deleted) def create_doc(self, content, doc_id=None): """ @@ -426,11 +408,13 @@ class Soledad(object): :param doc_id: an optional identifier specifying the document id :type doc_id: str - :return: the new document - :rtype: SoledadDocument + :return: + A deferred tht will fire with the new document (SoledadDocument + instance). + :rtype: Deferred """ - return self._db.create_doc( - self._convert_to_unicode(content), doc_id=doc_id) + return self._dbpool.create_doc( + _convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): """ @@ -446,10 +430,12 @@ class Soledad(object): :type json: str :param doc_id: An optional identifier specifying the document id. :type doc_id: - :return: The new document - :rtype: SoledadDocument + :return: + A deferred that will fire with the new document (A SoledadDocument + instance) + :rtype: Deferred """ - return self._db.create_doc_from_json(json, doc_id=doc_id) + return self._dbpool.create_doc_from_json(json, doc_id=doc_id) def create_index(self, index_name, *index_expressions): """ @@ -462,8 +448,8 @@ class Soledad(object): :param index_name: A unique name which can be used as a key prefix :type index_name: str - :param index_expressions: index expressions defining the index - information. + :param index_expressions: + index expressions defining the index information. :type index_expressions: dict Examples: @@ -473,9 +459,7 @@ class Soledad(object): "number(fieldname, width)", "lower(fieldname)" """ - if self._db: - return self._db.create_index( - index_name, *index_expressions) + return self._dbpool.create_index(index_name, *index_expressions) def delete_index(self, index_name): """ @@ -484,8 +468,7 @@ class Soledad(object): :param index_name: The name of the index we are removing :type index_name: str """ - if self._db: - return self._db.delete_index(index_name) + return self._dbpool.delete_index(index_name) def list_indexes(self): """ @@ -494,8 +477,7 @@ class Soledad(object): :return: A list of [('index-name', ['field', 'field2'])] definitions. :rtype: list """ - if self._db: - return self._db.list_indexes() + return self._dbpool.list_indexes() def get_from_index(self, index_name, *key_values): """ @@ -517,8 +499,7 @@ class Soledad(object): :return: List of [Document] :rtype: list """ - if self._db: - return self._db.get_from_index(index_name, *key_values) + return self._dbpool.get_from_index(index_name, *key_values) def get_count_from_index(self, index_name, *key_values): """ @@ -534,8 +515,7 @@ class Soledad(object): :return: count. :rtype: int """ - if self._db: - return self._db.get_count_from_index(index_name, *key_values) + return self._dbpool.get_count_from_index(index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): """ @@ -561,12 +541,11 @@ class Soledad(object): range. eg, if you have an index with 3 fields then you would have: (val1, val2, val3) :type end_values: tuple - :return: List of [Document] - :rtype: list + :return: A deferred that will fire with a list of [Document] + :rtype: Deferred """ - if self._db: - return self._db.get_range_from_index( - index_name, start_value, end_value) + return self._dbpool.get_range_from_index( + index_name, start_value, end_value) def get_index_keys(self, index_name): """ @@ -574,11 +553,11 @@ class Soledad(object): :param index_name: The index to query :type index_name: str - :return: [] A list of tuples of indexed keys. - :rtype: list + :return: + A deferred that will fire with a list of tuples of indexed keys. + :rtype: Deferred """ - if self._db: - return self._db.get_index_keys(index_name) + return self._dbpool.get_index_keys(index_name) def get_doc_conflicts(self, doc_id): """ @@ -587,11 +566,12 @@ class Soledad(object): :param doc_id: the document id :type doc_id: str - :return: a list of the document entries that are conflicted - :rtype: list + :return: + A deferred that will fire with a list of the document entries that + are conflicted. + :rtype: Deferred """ - if self._db: - return self._db.get_doc_conflicts(doc_id) + return self._dbpool.get_doc_conflicts(doc_id) def resolve_doc(self, doc, conflicted_doc_revs): """ @@ -599,12 +579,18 @@ class Soledad(object): :param doc: a document with the new content to be inserted. :type doc: SoledadDocument - :param conflicted_doc_revs: a list of revisions that the new content - supersedes. + :param conflicted_doc_revs: + A deferred that will fire with a list of revisions that the new + content supersedes. :type conflicted_doc_revs: list """ - if self._db: - return self._db.resolve_doc(doc, conflicted_doc_revs) + return self._dbpool.resolve_doc(doc, conflicted_doc_revs) + + # + # Sync API + # + + # TODO have interfaces, and let it implement it. def sync(self, defer_decryption=True): """ @@ -616,33 +602,38 @@ class Soledad(object): :param url: the url of the target replica to sync with :type url: str - :param defer_decryption: Whether to defer the decryption process using - the intermediate database. If False, - decryption will be done inline. + :param defer_decryption: + Whether to defer the decryption process using the intermediate + database. If False, decryption will be done inline. :type defer_decryption: bool - :return: The local generation before the synchronisation was - performed. + :return: + A deferred that will fire with the local generation before the + synchronisation was performed. :rtype: str """ - if self._db: - try: - local_gen = self._db.sync( - urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), - creds=self._creds, autocreate=False, - defer_decryption=defer_decryption) - soledad_events.signal( - soledad_events.SOLEDAD_DONE_DATA_SYNC, self._uuid) - return local_gen - except Exception as e: - logger.error("Soledad exception when syncing: %s" % str(e)) + # TODO this needs work. + # Should: + # (1) Defer to the syncer pool + # (2) Return a deferred (the deferToThreadpool can be good) + # (3) Add the callback for signaling the event + # (4) Let the local gen be returned from the thread + try: + local_gen = self._dbsyncer.sync( + urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), + creds=self._creds, autocreate=False, + defer_decryption=defer_decryption) + soledad_events.signal( + soledad_events.SOLEDAD_DONE_DATA_SYNC, self._uuid) + return local_gen + except Exception as e: + logger.error("Soledad exception when syncing: %s" % str(e)) def stop_sync(self): """ Stop the current syncing process. """ - if self._db: - self._db.stop_sync() + self._dbsyncer.stop_sync() def need_sync(self, url): """ @@ -654,12 +645,18 @@ class Soledad(object): :return: Whether remote replica and local replica differ. :rtype: bool """ + # XXX pass the get_replica_uid ------------------------ + # From where? initialize with that? + replica_uid = self._db._get_replica_uid() target = SoledadSyncTarget( - url, self._db._get_replica_uid(), creds=self._creds, - crypto=self._crypto) - info = target.get_sync_info(self._db._get_replica_uid()) + url, replica_uid, creds=self._creds, crypto=self._crypto) + + generation = self._db._get_generation() + # XXX better unpack it? + info = target.get_sync_info(replica_uid) + # compare source generation with target's last known source generation - if self._db._get_generation() != info[4]: + if generation != info[4]: soledad_events.signal( soledad_events.SOLEDAD_NEW_DATA_TO_SYNC, self._uuid) return True @@ -670,7 +667,7 @@ class Soledad(object): """ Property, True if the syncer is syncing. """ - return self._db.syncing + return self._dbsyncer.syncing def _set_token(self, token): """ @@ -781,6 +778,39 @@ class Soledad(object): self._secrets.change_passphrase(new_passphrase) +def _convert_to_unicode(content): + """ + Convert content to unicode (or all the strings in content) + + NOTE: Even though this method supports any type, it will + currently ignore contents of lists, tuple or any other + iterable than dict. We don't need support for these at the + moment + + :param content: content to convert + :type content: object + + :rtype: object + """ + if isinstance(content, unicode): + return content + elif isinstance(content, str): + result = chardet.detect(content) + default = "utf-8" + encoding = result["encoding"] or default + try: + content = content.decode(encoding) + except UnicodeError as e: + logger.error("Unicode error: {0!r}. Using 'replace'".format(e)) + content = content.decode(encoding, 'replace') + return content + else: + if isinstance(content, dict): + for key in content.keys(): + content[key] = _convert_to_unicode(content[key]) + return content + + # ---------------------------------------------------------------------------- # Monkey patching u1db to be able to provide a custom SSL cert # ---------------------------------------------------------------------------- @@ -819,4 +849,3 @@ class VerifiedHTTPSConnection(httplib.HTTPSConnection): old__VerifiedHTTPSConnection = http_client._VerifiedHTTPSConnection http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection - diff --git a/client/src/leap/soledad/client/examples/README b/client/src/leap/soledad/client/examples/README new file mode 100644 index 00000000..3aed8377 --- /dev/null +++ b/client/src/leap/soledad/client/examples/README @@ -0,0 +1,4 @@ +Right now, you can find here both an example of use +and the benchmarking scripts. +TODO move benchmark scripts to root scripts/ folder, +and leave here only a minimal example. diff --git a/client/src/leap/soledad/client/examples/compare.txt b/client/src/leap/soledad/client/examples/compare.txt new file mode 100644 index 00000000..19a1325a --- /dev/null +++ b/client/src/leap/soledad/client/examples/compare.txt @@ -0,0 +1,8 @@ +TIMES=100 TMPDIR=/media/sdb5/leap python use_adbapi.py 1.34s user 0.16s system 53% cpu 2.832 total +TIMES=100 TMPDIR=/media/sdb5/leap python use_api.py 1.22s user 0.14s system 62% cpu 2.181 total + +TIMES=1000 TMPDIR=/media/sdb5/leap python use_api.py 2.18s user 0.34s system 27% cpu 9.213 total +TIMES=1000 TMPDIR=/media/sdb5/leap python use_adbapi.py 2.40s user 0.34s system 39% cpu 7.004 total + +TIMES=5000 TMPDIR=/media/sdb5/leap python use_api.py 6.63s user 1.27s system 13% cpu 57.882 total +TIMES=5000 TMPDIR=/media/sdb5/leap python use_adbapi.py 6.84s user 1.26s system 36% cpu 22.367 total diff --git a/client/src/leap/soledad/client/examples/manifest.phk b/client/src/leap/soledad/client/examples/manifest.phk new file mode 100644 index 00000000..2c86c07d --- /dev/null +++ b/client/src/leap/soledad/client/examples/manifest.phk @@ -0,0 +1,50 @@ +The Hacker's Manifesto + +The Hacker's Manifesto +by: The Mentor + +Another one got caught today, it's all over the papers. "Teenager +Arrested in Computer Crime Scandal", "Hacker Arrested after Bank +Tampering." "Damn kids. They're all alike." But did you, in your +three-piece psychology and 1950's technobrain, ever take a look behind +the eyes of the hacker? Did you ever wonder what made him tick, what +forces shaped him, what may have molded him? I am a hacker, enter my +world. Mine is a world that begins with school. I'm smarter than most of +the other kids, this crap they teach us bores me. "Damn underachiever. +They're all alike." I'm in junior high or high school. I've listened to +teachers explain for the fifteenth time how to reduce a fraction. I +understand it. "No, Ms. Smith, I didn't show my work. I did it in +my head." "Damn kid. Probably copied it. They're all alike." I made a +discovery today. I found a computer. Wait a second, this is cool. It does +what I want it to. If it makes a mistake, it's because I screwed it up. +Not because it doesn't like me, or feels threatened by me, or thinks I'm +a smart ass, or doesn't like teaching and shouldn't be here. Damn kid. +All he does is play games. They're all alike. And then it happened... a +door opened to a world... rushing through the phone line like heroin +through an addict's veins, an electronic pulse is sent out, a refuge from +the day-to-day incompetencies is sought... a board is found. "This is +it... this is where I belong..." I know everyone here... even if I've +never met them, never talked to them, may never hear from them again... I +know you all... Damn kid. Tying up the phone line again. They're all +alike... You bet your ass we're all alike... we've been spoon-fed baby +food at school when we hungered for steak... the bits of meat that you +did let slip through were pre-chewed and tasteless. We've been dominated +by sadists, or ignored by the apathetic. The few that had something to +teach found us willing pupils, but those few are like drops of water in +the desert. This is our world now... the world of the electron and the +switch, the beauty of the baud. We make use of a service already existing +without paying for what could be dirt-cheap if it wasn't run by +profiteering gluttons, and you call us criminals. We explore... and you +call us criminals. We seek after knowledge... and you call us criminals. +We exist without skin color, without nationality, without religious +bias... and you call us criminals. You build atomic bombs, you wage wars, +you murder, cheat, and lie to us and try to make us believe it's for our +own good, yet we're the criminals. Yes, I am a criminal. My crime is that +of curiosity. My crime is that of judging people by what they say and +think, not what they look like. My crime is that of outsmarting you, +something that you will never forgive me for. I am a hacker, and this is +my manifesto. You may stop this individual, but you can't stop us all... +after all, we're all alike. + +This was the last published file written by The Mentor. Shortly after +releasing it, he was busted by the FBI. The Mentor, sadly missed. diff --git a/client/src/leap/soledad/client/examples/plot-async-db.py b/client/src/leap/soledad/client/examples/plot-async-db.py new file mode 100644 index 00000000..018a1a1d --- /dev/null +++ b/client/src/leap/soledad/client/examples/plot-async-db.py @@ -0,0 +1,45 @@ +import csv +from matplotlib import pyplot as plt + +FILE = "bench.csv" + +# config the plot +plt.xlabel('number of inserts') +plt.ylabel('time (seconds)') +plt.title('SQLCipher parallelization') + +kwargs = { + 'linewidth': 1.0, + 'linestyle': '-', +} + +series = (('sync', 'r'), + ('async', 'g')) + +data = {'mark': [], + 'sync': [], + 'async': []} + +with open(FILE, 'rb') as csvfile: + series_reader = csv.reader(csvfile, delimiter=',') + for m, s, a in series_reader: + data['mark'].append(int(m)) + data['sync'].append(float(s)) + data['async'].append(float(a)) + +xmax = max(data['mark']) +xmin = min(data['mark']) +ymax = max(data['sync'] + data['async']) +ymin = min(data['sync'] + data['async']) + +for run in series: + name = run[0] + color = run[1] + plt.plot(data['mark'], data[name], label=name, color=color, **kwargs) + +plt.axes().annotate("", xy=(xmax, ymax)) +plt.axes().annotate("", xy=(xmin, ymin)) + +plt.grid() +plt.legend() +plt.show() diff --git a/client/src/leap/soledad/client/examples/run_benchmark.py b/client/src/leap/soledad/client/examples/run_benchmark.py new file mode 100644 index 00000000..a112cf45 --- /dev/null +++ b/client/src/leap/soledad/client/examples/run_benchmark.py @@ -0,0 +1,28 @@ +""" +Run a mini-benchmark between regular api and dbapi +""" +import commands +import os +import time + +TMPDIR = os.environ.get("TMPDIR", "/tmp") +CSVFILE = 'bench.csv' + +cmd = "SILENT=1 TIMES={times} TMPDIR={tmpdir} python ./use_{version}api.py" + +parse_time = lambda r: r.split('\n')[-1] + + +with open(CSVFILE, 'w') as log: + + for times in range(0, 10000, 500): + cmd1 = cmd.format(times=times, tmpdir=TMPDIR, version="") + sync_time = parse_time(commands.getoutput(cmd1)) + + cmd2 = cmd.format(times=times, tmpdir=TMPDIR, version="adb") + async_time = parse_time(commands.getoutput(cmd2)) + + print times, sync_time, async_time + log.write("%s, %s, %s\n" % (times, sync_time, async_time)) + log.flush() + time.sleep(2) diff --git a/client/src/leap/soledad/client/examples/use_adbapi.py b/client/src/leap/soledad/client/examples/use_adbapi.py new file mode 100644 index 00000000..d3ee8527 --- /dev/null +++ b/client/src/leap/soledad/client/examples/use_adbapi.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# use_adbapi.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Example of use of the asynchronous soledad api. +""" +from __future__ import print_function +import datetime +import os + +import u1db +from twisted.internet import defer, reactor + +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +folder = os.environ.get("TMPDIR", "tmp") +times = int(os.environ.get("TIMES", "1000")) +silent = os.environ.get("SILENT", False) + +tmpdb = os.path.join(folder, "test.soledad") + + +def debug(*args): + if not silent: + print(*args) + +debug("[+] db path:", tmpdb) +debug("[+] times", times) + +if os.path.isfile(tmpdb): + debug("[+] Removing existing db file...") + os.remove(tmpdb) + +start_time = datetime.datetime.now() + +opts = SQLCipherOptions(tmpdb, "secret", create=True) +dbpool = adbapi.getConnectionPool(opts) + + +def createDoc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + + +def getAllDocs(): + return dbpool.runU1DBQuery("get_all_docs") + + +def countDocs(_): + debug("counting docs...") + d = getAllDocs() + d.addCallbacks(printResult, lambda e: e.printTraceback()) + d.addBoth(allDone) + + +def printResult(r): + if isinstance(r, u1db.Document): + debug(r.doc_id, r.content['number']) + else: + len_results = len(r[1]) + debug("GOT %s results" % len(r[1])) + + if len_results == times: + debug("ALL GOOD") + else: + raise ValueError("We didn't expect this result len") + + +def allDone(_): + debug("ALL DONE!") + if silent: + end_time = datetime.datetime.now() + print((end_time - start_time).total_seconds()) + reactor.stop() + +deferreds = [] + +for i in range(times): + doc = {"number": i, + "payload": open('manifest.phk').read()} + d = createDoc(doc) + d.addCallbacks(printResult, lambda e: e.printTraceback()) + deferreds.append(d) + + +all_done = defer.gatherResults(deferreds, consumeErrors=True) +all_done.addCallback(countDocs) + +reactor.run() diff --git a/client/src/leap/soledad/client/examples/use_api.py b/client/src/leap/soledad/client/examples/use_api.py new file mode 100644 index 00000000..fd0a100c --- /dev/null +++ b/client/src/leap/soledad/client/examples/use_api.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# use_api.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Example of use of the soledad api. +""" +from __future__ import print_function +import datetime +import os + +from leap.soledad.client import sqlcipher +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +folder = os.environ.get("TMPDIR", "tmp") +times = int(os.environ.get("TIMES", "1000")) +silent = os.environ.get("SILENT", False) + +tmpdb = os.path.join(folder, "test.soledad") + + +def debug(*args): + if not silent: + print(*args) + +debug("[+] db path:", tmpdb) +debug("[+] times", times) + +if os.path.isfile(tmpdb): + debug("[+] Removing existing db file...") + os.remove(tmpdb) + +start_time = datetime.datetime.now() + +opts = SQLCipherOptions(tmpdb, "secret", create=True) +db = sqlcipher.SQLCipherDatabase(None, opts) + + +def allDone(): + debug("ALL DONE!") + + +for i in range(times): + doc = {"number": i, + "payload": open('manifest.phk').read()} + d = db.create_doc(doc) + debug(d.doc_id, d.content['number']) + +debug("Count", len(db.get_all_docs()[1])) +if silent: + end_time = datetime.datetime.now() + print((end_time - start_time).total_seconds()) + +allDone() diff --git a/client/src/leap/soledad/client/mp_safe_db_TOREMOVE.py b/client/src/leap/soledad/client/mp_safe_db_TOREMOVE.py deleted file mode 100644 index 9ed0bef4..00000000 --- a/client/src/leap/soledad/client/mp_safe_db_TOREMOVE.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -# mp_safe_db.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Multiprocessing-safe SQLite database. -""" - - -from threading import Thread -from Queue import Queue -from pysqlcipher import dbapi2 - - -# Thanks to http://code.activestate.com/recipes/526618/ - -class MPSafeSQLiteDB(Thread): - """ - A multiprocessing-safe SQLite database accessor. - """ - - CLOSE = "--close--" - NO_MORE = "--no more--" - - def __init__(self, db_path): - """ - Initialize the process - """ - Thread.__init__(self) - self._db_path = db_path - self._requests = Queue() - self.start() - - def run(self): - """ - Run the multiprocessing-safe database accessor. - """ - conn = dbapi2.connect(self._db_path) - while True: - req, arg, res = self._requests.get() - if req == self.CLOSE: - break - with conn: - cursor = conn.cursor() - cursor.execute(req, arg) - if res: - for rec in cursor.fetchall(): - res.put(rec) - res.put(self.NO_MORE) - conn.close() - - def execute(self, req, arg=None, res=None): - """ - Execute a request on the database. - - :param req: The request to be executed. - :type req: str - :param arg: The arguments for the request. - :type arg: tuple - :param res: A queue to write request results. - :type res: multiprocessing.Queue - """ - self._requests.put((req, arg or tuple(), res)) - - def select(self, req, arg=None): - """ - Run a select query on the database and yield results. - - :param req: The request to be executed. - :type req: str - :param arg: The arguments for the request. - :type arg: tuple - """ - res = Queue() - self.execute(req, arg, res) - while True: - rec = res.get() - if rec == self.NO_MORE: - break - yield rec - - def close(self): - """ - Close the database connection. - """ - self.execute(self.CLOSE) - self.join() - - def cursor(self): - """ - Return a fake cursor object. - - Not really a cursor, but allows for calling db.cursor().execute(). - - :return: Self. - :rtype: MPSafeSQLiteDatabase - """ - return self diff --git a/client/src/leap/soledad/client/pragmas.py b/client/src/leap/soledad/client/pragmas.py index a21e68a8..7a13a694 100644 --- a/client/src/leap/soledad/client/pragmas.py +++ b/client/src/leap/soledad/client/pragmas.py @@ -15,18 +15,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Different pragmas used in the SQLCIPHER database. +Different pragmas used in the initialization of the SQLCipher database. """ -# TODO --------------------------------------------------------------- -# Work In Progress. -# We need to reduce the impedance mismatch between the current soledad -# implementation and the eventually asynchronous api. -# So... how to plug it in, allowing for an optional sync / async coexistence? -# One of the first things is to isolate all the pragmas work that has to be -# done during initialization. -# And, instead of having all of them passed the db_handle and executing that, -# we could have just a string returned, that can be chained to a deferred. -# --------------------------------------------------------------------- import logging import string @@ -81,7 +71,7 @@ def _set_key(db_handle, key, is_raw_key): _set_key_passphrase(db_handle, key) -def _set_key_passphrase(cls, db_handle, passphrase): +def _set_key_passphrase(db_handle, passphrase): """ Set a passphrase for encryption key derivation. @@ -265,7 +255,7 @@ def _set_rekey_passphrase(db_handle, passphrase): db_handle.cursor().execute("PRAGMA rekey = '%s'" % passphrase) -def _set_rekey_raw(cls, db_handle, key): +def _set_rekey_raw(db_handle, key): """ Change the raw hexadecimal encryption key. @@ -300,7 +290,7 @@ def set_synchronous_normal(db_handle): db_handle.cursor().execute('PRAGMA synchronous=NORMAL') -def set_mem_temp_store(cls, db_handle): +def set_mem_temp_store(db_handle): """ Use a in-memory store for temporary tables. """ @@ -308,7 +298,7 @@ def set_mem_temp_store(cls, db_handle): db_handle.cursor().execute('PRAGMA temp_store=MEMORY') -def set_write_ahead_logging(cls, db_handle): +def set_write_ahead_logging(db_handle): """ Enable write-ahead logging, and set the autocheckpoint to 50 pages. diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index fcef592d..c9e69c73 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -45,7 +45,7 @@ import logging import multiprocessing import os import threading -import time +# import time --- needed for the win initialization hack import json from hashlib import sha256 @@ -58,11 +58,13 @@ from u1db.backends import sqlite_backend from u1db import errors as u1db_errors from taskthread import TimerTask -from leap.soledad.client.crypto import SyncEncrypterPool, SyncDecrypterPool +from leap.soledad.client import crypto from leap.soledad.client.target import SoledadSyncTarget from leap.soledad.client.target import PendingReceivedDocsSyncError from leap.soledad.client.sync import SoledadSynchronizer -from leap.soledad.client.mp_safe_db import MPSafeSQLiteDB + +# TODO use adbapi too +from leap.soledad.client.mp_safe_db_TOREMOVE import MPSafeSQLiteDB from leap.soledad.client import pragmas from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument @@ -80,36 +82,81 @@ sqlite_backend.dbapi2 = sqlcipher_dbapi2 # See https://sqlite.org/threadsafe.html # and http://bugs.python.org/issue16509 -SQLITE_CHECK_SAME_THREAD = False +# TODO this no longer needed ------------- +#SQLITE_CHECK_SAME_THREAD = False + + +def initialize_sqlcipher_db(opts, on_init=None): + """ + Initialize a SQLCipher database. + + :param opts: + :type opts: SQLCipherOptions + :param on_init: a tuple of queries to be executed on initialization + :type on_init: tuple + :return: a SQLCipher connection + """ + conn = sqlcipher_dbapi2.connect( + opts.path) + + # XXX not needed -- check + #check_same_thread=SQLITE_CHECK_SAME_THREAD) + + set_init_pragmas(conn, opts, extra_queries=on_init) + return conn + +_db_init_lock = threading.Lock() + + +def set_init_pragmas(conn, opts=None, extra_queries=None): + """ + Set the initialization pragmas. + + This includes the crypto pragmas, and any other options that must + be passed early to sqlcipher db. + """ + assert opts is not None + extra_queries = [] if extra_queries is None else extra_queries + with _db_init_lock: + # only one execution path should initialize the db + _set_init_pragmas(conn, opts, extra_queries) + + +def _set_init_pragmas(conn, opts, extra_queries): -# We set isolation_level to None to setup autocommit mode. -# See: http://docs.python.org/2/library/sqlite3.html#controlling-transactions -# This avoids problems with sequential operations using the same soledad object -# trying to open new transactions -# (The error was: -# OperationalError:cannot start a transaction within a transaction.) -SQLITE_ISOLATION_LEVEL = None + sync_off = os.environ.get('LEAP_SQLITE_NOSYNC') + memstore = os.environ.get('LEAP_SQLITE_MEMSTORE') + nowal = os.environ.get('LEAP_SQLITE_NOWAL') + + pragmas.set_crypto_pragmas(conn, opts) + + if not nowal: + pragmas.set_write_ahead_logging(conn) + if sync_off: + pragmas.set_synchronous_off(conn) + else: + pragmas.set_synchronous_normal(conn) + if memstore: + pragmas.set_mem_temp_store(conn) + + for query in extra_queries: + conn.cursor().execute(query) -# TODO accept cyrpto object too.... or pass it along.. class SQLCipherOptions(object): + """ + A container with options for the initialization of an SQLCipher database. + """ def __init__(self, path, key, create=True, is_raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, - document_factory=None, defer_encryption=False, sync_db_key=None): """ - Options for the initialization of an SQLCipher database. - :param path: The filesystem path for the database to open. :type path: str :param create: True/False, should the database be created if it doesn't already exist? :param create: bool - :param document_factory: - A function that will be called with the same parameters as - Document.__init__. - :type document_factory: callable :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt document contents when syncing. :type crypto: soledad.crypto.SoledadCrypto @@ -137,87 +184,22 @@ class SQLCipherOptions(object): self.cipher_page_size = cipher_page_size self.defer_encryption = defer_encryption self.sync_db_key = sync_db_key - self.document_factory = None - - -# XXX Use SQLCIpherOptions instead -#def open(path, password, create=True, document_factory=None, crypto=None, - #raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, - #cipher_page_size=1024, defer_encryption=False, sync_db_key=None): - #""" - #Open a database at the given location. -# - #*** IMPORTANT *** -# - #Don't forget to close the database after use by calling the close() - #method otherwise some resources might not be freed and you may experience - #several kinds of leakages. -# - #*** IMPORTANT *** -# - #Will raise u1db.errors.DatabaseDoesNotExist if create=False and the - #database does not already exist. -# - #:return: An instance of Database. - #:rtype SQLCipherDatabase - #""" - #args = (path, password) - #kwargs = { - #'create': create, - #'document_factory': document_factory, - #'crypto': crypto, - #'raw_key': raw_key, - #'cipher': cipher, - #'kdf_iter': kdf_iter, - #'cipher_page_size': cipher_page_size, - #'defer_encryption': defer_encryption, - #'sync_db_key': sync_db_key} - # XXX pass only a CryptoOptions object around - #return SQLCipherDatabase.open_database(*args, **kwargs) - # # The SQLCipher database # + class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ A U1DB implementation that uses SQLCipher as its persistence layer. """ defer_encryption = False - _index_storage_value = 'expand referenced encrypted' - k_lock = threading.Lock() - create_doc_lock = threading.Lock() - update_indexes_lock = threading.Lock() - _sync_watcher = None - _sync_enc_pool = None - - """ - The name of the local symmetrically encrypted documents to - sync database file. - """ - LOCAL_SYMMETRIC_SYNC_FILE_NAME = 'sync.u1db' - - """ - A dictionary that hold locks which avoid multiple sync attempts from the - same database replica. - """ - encrypting_lock = threading.Lock() - - """ - Period or recurrence of the periodic encrypting task, in seconds. - """ - ENCRYPT_TASK_PERIOD = 1 + # XXX not used afaik: + # _index_storage_value = 'expand referenced encrypted' - syncing_lock = defaultdict(threading.Lock) - """ - A dictionary that hold locks which avoid multiple sync attempts from the - same database replica. - """ - - # XXX Use SQLCIpherOptions instead - def __init__(self, opts): + def __init__(self, soledad_crypto, opts): """ Connect to an existing SQLCipher database, creating a new sqlcipher database file if needed. @@ -230,76 +212,23 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): *** IMPORTANT *** + :param soledad_crypto: + :type soldead_crypto: :param opts: :type opts: SQLCipherOptions """ + # TODO ------ we don't need any soledad crypto in here + # ensure the db is encrypted if the file already exists - if os.path.exists(opts.sqlcipher_file): + if os.path.isfile(opts.path): self.assert_db_is_encrypted(opts) # connect to the sqlcipher database - # XXX this lock should not be needed ----------------- - # u1db holds a mutex over sqlite internally for the initialization. - with self.k_lock: - self._db_handle = sqlcipher_dbapi2.connect( - - # TODO ----------------------------------------------- - # move the init to a single function - opts.sqlcipher_file, - isolation_level=SQLITE_ISOLATION_LEVEL, - check_same_thread=SQLITE_CHECK_SAME_THREAD) - # set SQLCipher cryptographic parameters - - # XXX allow optional deferredChain here ? - pragmas.set_crypto_pragmas( - self._db_handle, password, raw_key, cipher, kdf_iter, - cipher_page_size) - if os.environ.get('LEAP_SQLITE_NOSYNC'): - pragmas.set_synchronous_off(self._db_handle) - else: - pragmas.set_synchronous_normal(self._db_handle) - if os.environ.get('LEAP_SQLITE_MEMSTORE'): - pragmas.set_mem_temp_store(self._db_handle) - pragmas.set_write_ahead_logging(self._db_handle) - - self._real_replica_uid = None - self._ensure_schema() - self._crypto = opts.crypto - - - # TODO ------------------------------------------------ - # Move syncdb to another class ------------------------ - # define sync-db attrs - self._sqlcipher_file = sqlcipher_file - self._sync_db_key = sync_db_key - self._sync_db = None - self._sync_db_write_lock = None - self._sync_enc_pool = None - self.sync_queue = None + self._db_handle = initialize_sqlcipher_db(opts) + self._real_replica_uid = None + self._ensure_schema() - if self.defer_encryption: - # initialize sync db - self._init_sync_db() - # initialize syncing queue encryption pool - self._sync_enc_pool = SyncEncrypterPool( - self._crypto, self._sync_db, self._sync_db_write_lock) - self._sync_watcher = TimerTask(self._encrypt_syncing_docs, - self.ENCRYPT_TASK_PERIOD) - self._sync_watcher.start() - - def factory(doc_id=None, rev=None, json='{}', has_conflicts=False, - syncable=True): - return SoledadDocument(doc_id=doc_id, rev=rev, json=json, - has_conflicts=has_conflicts, - syncable=syncable) - self.set_document_factory(factory) - # we store syncers in a dictionary indexed by the target URL. We also - # store a hash of the auth info in case auth info expires and we need - # to rebuild the syncer for that target. The final self._syncers - # format is the following: - # - # self._syncers = {'': ('', syncer), ...} - self._syncers = {} + self.set_document_factory(soledad_doc_factory) def _extra_schema_init(self, c): """ @@ -312,40 +241,212 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :param c: The cursor for querying the database. :type c: dbapi2.cursor """ + print "CALLING EXTRA SCHEMA INIT...." c.execute( 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') + # + # Document operations + # + + def put_doc(self, doc): + """ + Overwrite the put_doc method, to enqueue the modified document for + encryption before sync. + + :param doc: The document to be put. + :type doc: u1db.Document + + :return: The new document revision. + :rtype: str + """ + doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc) + + # XXX move to API + if self.defer_encryption: + self.sync_queue.put_nowait(doc) + return doc_rev + + # + # SQLCipher API methods + # + + # TODO this doesn't need to be an instance method + def assert_db_is_encrypted(self, opts): + """ + Assert that the sqlcipher file contains an encrypted database. + + When opening an existing database, PRAGMA key will not immediately + throw an error if the key provided is incorrect. To test that the + database can be successfully opened with the provided key, it is + necessary to perform some operation on the database (i.e. read from + it) and confirm it is success. + + The easiest way to do this is select off the sqlite_master table, + which will attempt to read the first page of the database and will + parse the schema. + + :param opts: + """ + # We try to open an encrypted database with the regular u1db + # backend should raise a DatabaseError exception. + # If the regular backend succeeds, then we need to stop because + # the database was not properly initialized. + try: + sqlite_backend.SQLitePartialExpandDatabase(opts.path) + except sqlcipher_dbapi2.DatabaseError: + # assert that we can access it using SQLCipher with the given + # key + dummy_query = ('SELECT count(*) FROM sqlite_master',) + initialize_sqlcipher_db(opts, on_init=dummy_query) + else: + raise DatabaseIsNotEncrypted() + + # Extra query methods: extensions to the base u1db sqlite implmentation. + + def get_count_from_index(self, index_name, *key_values): + """ + Return the count for a given combination of index_name + and key values. + + Extension method made from similar methods in u1db version 13.09 + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: count. + :rtype: int + """ + c = self._db_handle.cursor() + definition = self._get_index_definition(index_name) + + if len(key_values) != len(definition): + raise u1db_errors.InvalidValueForIndex() + tables = ["document_fields d%d" % i for i in range(len(definition))] + novalue_where = ["d.doc_id = d%d.doc_id" + " AND d%d.field_name = ?" + % (i, i) for i in range(len(definition))] + exact_where = [novalue_where[i] + + (" AND d%d.value = ?" % (i,)) + for i in range(len(definition))] + args = [] + where = [] + for idx, (field, value) in enumerate(zip(definition, key_values)): + args.append(field) + where.append(exact_where[idx]) + args.append(value) + + tables = ["document_fields d%d" % i for i in range(len(definition))] + statement = ( + "SELECT COUNT(*) FROM document d, %s WHERE %s " % ( + ', '.join(tables), + ' AND '.join(where), + )) + try: + c.execute(statement, tuple(args)) + except sqlcipher_dbapi2.OperationalError, e: + raise sqlcipher_dbapi2.OperationalError( + str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) + res = c.fetchall() + return res[0][0] + + def close(self): + """ + Close db connections. + """ + # TODO should be handled by adbapi instead + # TODO syncdb should be stopped first + + if logger is not None: # logger might be none if called from __del__ + logger.debug("SQLCipher backend: closing") + + # close the actual database + if self._db_handle is not None: + self._db_handle.close() + self._db_handle = None + + # indexes + + def _put_and_update_indexes(self, old_doc, doc): + """ + Update a document and all indexes related to it. + + :param old_doc: The old version of the document. + :type old_doc: u1db.Document + :param doc: The new version of the document. + :type doc: u1db.Document + """ + sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( + self, old_doc, doc) + c = self._db_handle.cursor() + c.execute('UPDATE document SET syncable=? WHERE doc_id=?', + (doc.syncable, doc.doc_id)) + + def _get_doc(self, doc_id, check_for_conflicts=False): + """ + Get just the document content, without fancy handling. + + :param doc_id: The unique document identifier + :type doc_id: str + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise asking for a deleted + document will return None. + :type include_deleted: bool + + :return: a Document object. + :type: u1db.Document + """ + doc = sqlite_backend.SQLitePartialExpandDatabase._get_doc( + self, doc_id, check_for_conflicts) + if doc: + c = self._db_handle.cursor() + c.execute('SELECT syncable FROM document WHERE doc_id=?', + (doc.doc_id,)) + result = c.fetchone() + doc.syncable = bool(result[0]) + return doc + + def __del__(self): + """ + Free resources when deleting or garbage collecting the database. + + This is only here to minimze problems if someone ever forgets to call + the close() method after using the database; you should not rely on + garbage collecting to free up the database resources. + """ + self.close() # TODO ---- rescue the fix for the windows case from here... - #@classmethod - # XXX Use SQLCIpherOptions instead - #def _open_database(cls, sqlcipher_file, password, document_factory=None, - #crypto=None, raw_key=False, cipher='aes-256-cbc', - #kdf_iter=4000, cipher_page_size=1024, - #defer_encryption=False, sync_db_key=None): - #""" - #Open a SQLCipher database. + # @classmethod + # def _open_database(cls, sqlcipher_file, password, document_factory=None, + # crypto=None, raw_key=False, cipher='aes-256-cbc', + # kdf_iter=4000, cipher_page_size=1024, + # defer_encryption=False, sync_db_key=None): + # """ + # Open a SQLCipher database. # - #:return: The database object. - #:rtype: SQLCipherDatabase - #""" - #cls.defer_encryption = defer_encryption - #if not os.path.isfile(sqlcipher_file): - #raise u1db_errors.DatabaseDoesNotExist() + # :return: The database object. + # :rtype: SQLCipherDatabase + # """ + # cls.defer_encryption = defer_encryption + # if not os.path.isfile(sqlcipher_file): + # raise u1db_errors.DatabaseDoesNotExist() # - #tries = 2 + # tries = 2 # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6) # where without re-opening the database on Windows, it # doesn't see the transaction that was just committed - #while True: -# - #with cls.k_lock: - #db_handle = dbapi2.connect( - #sqlcipher_file, - #check_same_thread=SQLITE_CHECK_SAME_THREAD) + # while True: + # with cls.k_lock: + # db_handle = dbapi2.connect( + # sqlcipher_file, + # check_same_thread=SQLITE_CHECK_SAME_THREAD) # - #try: + # try: # set cryptographic params # # XXX pass only a CryptoOptions object around @@ -374,49 +475,108 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): #crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, #cipher_page_size=cipher_page_size, sync_db_key=sync_db_key) - #@classmethod - #def open_database(cls, sqlcipher_file, password, create, - #document_factory=None, crypto=None, raw_key=False, - #cipher='aes-256-cbc', kdf_iter=4000, - #cipher_page_size=1024, defer_encryption=False, - #sync_db_key=None): - # XXX pass only a CryptoOptions object around - #""" - #Open a SQLCipher database. -# - #*** IMPORTANT *** -# - #Don't forget to close the database after use by calling the close() - #method otherwise some resources might not be freed and you may - #experience several kinds of leakages. -# - #*** IMPORTANT *** -# - #:return: The database object. - #:rtype: SQLCipherDatabase - #""" - #cls.defer_encryption = defer_encryption - #args = sqlcipher_file, password - #kwargs = { - #'crypto': crypto, - #'raw_key': raw_key, - #'cipher': cipher, - #'kdf_iter': kdf_iter, - #'cipher_page_size': cipher_page_size, - #'defer_encryption': defer_encryption, - #'sync_db_key': sync_db_key, - #'document_factory': document_factory, - #} - #try: - #return cls._open_database(*args, **kwargs) - #except u1db_errors.DatabaseDoesNotExist: - #if not create: - #raise -# - # XXX here we were missing sync_db_key, intentional? - #return SQLCipherDatabase(*args, **kwargs) - # BEGIN SYNC FOO ---------------------------------------------------------- +class SQLCipherU1DBSync(object): + + _sync_watcher = None + _sync_enc_pool = None + + """ + The name of the local symmetrically encrypted documents to + sync database file. + """ + LOCAL_SYMMETRIC_SYNC_FILE_NAME = 'sync.u1db' + + """ + A dictionary that hold locks which avoid multiple sync attempts from the + same database replica. + """ + # XXX We do not need the lock here now. Remove. + encrypting_lock = threading.Lock() + + """ + Period or recurrence of the periodic encrypting task, in seconds. + """ + # XXX use LoopingCall. + # Just use fucking deferreds, do not waste time looping. + ENCRYPT_TASK_PERIOD = 1 + + """ + A dictionary that hold locks which avoid multiple sync attempts from the + same database replica. + """ + syncing_lock = defaultdict(threading.Lock) + + def _init_sync(self, opts, soledad_crypto, defer_encryption=False): + + self._crypto = soledad_crypto + + # TODO ----- have to decide what to do with syncer + self._sync_db_key = opts.sync_db_key + self._sync_db = None + self._sync_db_write_lock = None + self._sync_enc_pool = None + self.sync_queue = None + + if self.defer_encryption: + # initialize sync db + self._init_sync_db() + # initialize syncing queue encryption pool + self._sync_enc_pool = crypto.SyncEncrypterPool( + self._crypto, self._sync_db, self._sync_db_write_lock) + self._sync_watcher = TimerTask(self._encrypt_syncing_docs, + self.ENCRYPT_TASK_PERIOD) + self._sync_watcher.start() + + # TODO move to class attribute? + # we store syncers in a dictionary indexed by the target URL. We also + # store a hash of the auth info in case auth info expires and we need + # to rebuild the syncer for that target. The final self._syncers + # format is the following:: + # + # self._syncers = {'': ('', syncer), ...} + self._syncers = {} + self._sync_db_write_lock = threading.Lock() + self.sync_queue = multiprocessing.Queue() + + def _init_sync_db(self, opts): + """ + Initialize the Symmetrically-Encrypted document to be synced database, + and the queue to communicate with subprocess workers. + + :param opts: + :type opts: SQLCipherOptions + """ + soledad_assert(opts.sync_db_key is not None) + sync_db_path = None + if opts.path != ":memory:": + sync_db_path = "%s-sync" % opts.path + else: + sync_db_path = ":memory:" + + # XXX use initialize_sqlcipher_db here too + # TODO pass on_init queries to initialize_sqlcipher_db + self._sync_db = MPSafeSQLiteDB(sync_db_path) + pragmas.set_crypto_pragmas(self._sync_db, opts) + + # create sync tables + self._create_sync_db_tables() + + def _create_sync_db_tables(self): + """ + Create tables for the local sync documents db if needed. + """ + # TODO use adbapi --------------------------------- + encr = crypto.SyncEncrypterPool + decr = crypto.SyncDecrypterPool + sql_encr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % ( + encr.TABLE_NAME, encr.FIELD_NAMES)) + sql_decr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % ( + decr.TABLE_NAME, decr.FIELD_NAMES)) + + with self._sync_db_write_lock: + self._sync_db.execute(sql_encr) + self._sync_db.execute(sql_decr) def sync(self, url, creds=None, autocreate=True, defer_decryption=True): """ @@ -428,14 +588,15 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :param url: The url of the target replica to sync with. :type url: str - :param creds: optional dictionary giving credentials. + :param creds: + optional dictionary giving credentials. to authorize the operation with the server. :type creds: dict :param autocreate: Ask the target to create the db if non-existent. :type autocreate: bool - :param defer_decryption: Whether to defer the decryption process using - the intermediate database. If False, - decryption will be done inline. + :param defer_decryption: + Whether to defer the decryption process using the intermediate + database. If False, decryption will be done inline. :type defer_decryption: bool :return: The local generation before the synchronisation was performed. @@ -482,13 +643,13 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): Because of that, this method blocks until the syncing lock can be acquired. """ - with SQLCipherDatabase.syncing_lock[self._get_replica_uid()]: + with self.syncing_lock[self._get_replica_uid()]: syncer = self._get_syncer(url, creds=creds) yield syncer @property def syncing(self): - lock = SQLCipherDatabase.syncing_lock[self._get_replica_uid()] + lock = self.syncing_lock[self._get_replica_uid()] acquired_lock = lock.acquire(False) if acquired_lock is False: return True @@ -530,46 +691,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): syncer.num_inserted = 0 return syncer - # END SYNC FOO ---------------------------------------------------------- - - def _init_sync_db(self): - """ - Initialize the Symmetrically-Encrypted document to be synced database, - and the queue to communicate with subprocess workers. - """ - if self._sync_db is None: - soledad_assert(self._sync_db_key is not None) - sync_db_path = None - if self._sqlcipher_file != ":memory:": - sync_db_path = "%s-sync" % self._sqlcipher_file - else: - sync_db_path = ":memory:" - self._sync_db = MPSafeSQLiteDB(sync_db_path) - # protect the sync db with a password - if self._sync_db_key is not None: - # XXX pass only a CryptoOptions object around - pragmas.set_crypto_pragmas( - self._sync_db, self._sync_db_key, False, - 'aes-256-cbc', 4000, 1024) - self._sync_db_write_lock = threading.Lock() - self._create_sync_db_tables() - self.sync_queue = multiprocessing.Queue() - - def _create_sync_db_tables(self): - """ - Create tables for the local sync documents db if needed. - """ - encr = SyncEncrypterPool - decr = SyncDecrypterPool - sql_encr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % ( - encr.TABLE_NAME, encr.FIELD_NAMES)) - sql_decr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % ( - decr.TABLE_NAME, decr.FIELD_NAMES)) - - with self._sync_db_write_lock: - self._sync_db.execute(sql_encr) - self._sync_db.execute(sql_decr) - # # Symmetric encryption of syncing docs # @@ -599,182 +720,14 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): finally: lock.release() - # - # Document operations - # - - def put_doc(self, doc): - """ - Overwrite the put_doc method, to enqueue the modified document for - encryption before sync. - - :param doc: The document to be put. - :type doc: u1db.Document - - :return: The new document revision. - :rtype: str - """ - doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc) - if self.defer_encryption: - self.sync_queue.put_nowait(doc) - return doc_rev - - # indexes - - def _put_and_update_indexes(self, old_doc, doc): - """ - Update a document and all indexes related to it. - - :param old_doc: The old version of the document. - :type old_doc: u1db.Document - :param doc: The new version of the document. - :type doc: u1db.Document - """ - with self.update_indexes_lock: - sqlite_backend.SQLitePartialExpandDatabase._put_and_update_indexes( - self, old_doc, doc) - c = self._db_handle.cursor() - c.execute('UPDATE document SET syncable=? ' - 'WHERE doc_id=?', - (doc.syncable, doc.doc_id)) - - def _get_doc(self, doc_id, check_for_conflicts=False): - """ - Get just the document content, without fancy handling. - - :param doc_id: The unique document identifier - :type doc_id: str - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise asking for a deleted - document will return None. - :type include_deleted: bool - - :return: a Document object. - :type: u1db.Document - """ - doc = sqlite_backend.SQLitePartialExpandDatabase._get_doc( - self, doc_id, check_for_conflicts) - if doc: - c = self._db_handle.cursor() - c.execute('SELECT syncable FROM document WHERE doc_id=?', - (doc.doc_id,)) - result = c.fetchone() - doc.syncable = bool(result[0]) - return doc - - # - # SQLCipher API methods - # - - # XXX Use SQLCIpherOptions instead - @classmethod - def assert_db_is_encrypted(cls, sqlcipher_file, key, raw_key, cipher, - kdf_iter, cipher_page_size): - """ - Assert that C{sqlcipher_file} contains an encrypted database. - - When opening an existing database, PRAGMA key will not immediately - throw an error if the key provided is incorrect. To test that the - database can be successfully opened with the provided key, it is - necessary to perform some operation on the database (i.e. read from - it) and confirm it is success. - - The easiest way to do this is select off the sqlite_master table, - which will attempt to read the first page of the database and will - parse the schema. - - :param sqlcipher_file: The path for the SQLCipher file. - :type sqlcipher_file: str - :param key: The key that protects the SQLCipher db. - :type key: str - :param raw_key: Whether C{key} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the encyrption key. - :type raw_key: bool - :param cipher: The cipher and mode to use. - :type cipher: str - :param kdf_iter: The number of iterations to use. - :type kdf_iter: int - :param cipher_page_size: The page size. - :type cipher_page_size: int - """ - try: - # try to open an encrypted database with the regular u1db - # backend should raise a DatabaseError exception. - sqlite_backend.SQLitePartialExpandDatabase(sqlcipher_file) - raise DatabaseIsNotEncrypted() - except sqlcipher_dbapi2.DatabaseError: - # assert that we can access it using SQLCipher with the given - # key - with cls.k_lock: - db_handle = sqlcipher_dbapi2.connect( - sqlcipher_file, - isolation_level=SQLITE_ISOLATION_LEVEL, - check_same_thread=SQLITE_CHECK_SAME_THREAD) - pragmas.set_crypto_pragmas( - db_handle, key, raw_key, cipher, - kdf_iter, cipher_page_size) - db_handle.cursor().execute( - 'SELECT count(*) FROM sqlite_master') - - # Extra query methods: extensions to the base sqlite implmentation. - - def get_count_from_index(self, index_name, *key_values): - """ - Returns the count for a given combination of index_name - and key values. - - Extension method made from similar methods in u1db version 13.09 - - :param index_name: The index to query - :type index_name: str - :param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - :type key_values: tuple - :return: count. - :rtype: int - """ - c = self._db_handle.cursor() - definition = self._get_index_definition(index_name) - - if len(key_values) != len(definition): - raise u1db_errors.InvalidValueForIndex() - tables = ["document_fields d%d" % i for i in range(len(definition))] - novalue_where = ["d.doc_id = d%d.doc_id" - " AND d%d.field_name = ?" - % (i, i) for i in range(len(definition))] - exact_where = [novalue_where[i] - + (" AND d%d.value = ?" % (i,)) - for i in range(len(definition))] - args = [] - where = [] - for idx, (field, value) in enumerate(zip(definition, key_values)): - args.append(field) - where.append(exact_where[idx]) - args.append(value) - - tables = ["document_fields d%d" % i for i in range(len(definition))] - statement = ( - "SELECT COUNT(*) FROM document d, %s WHERE %s " % ( - ', '.join(tables), - ' AND '.join(where), - )) - try: - c.execute(statement, tuple(args)) - except sqlcipher_dbapi2.OperationalError, e: - raise sqlcipher_dbapi2.OperationalError( - str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) - res = c.fetchall() - return res[0][0] + @property + def replica_uid(self): + return self._get_replica_uid() def close(self): """ - Close db_handle and close syncer. + Close the syncer and syncdb orderly """ - # TODO separate db from syncers -------------- - - if logger is not None: # logger might be none if called from __del__ - logger.debug("Sqlcipher backend: closing") # stop the sync watcher for deferred encryption if self._sync_watcher is not None: self._sync_watcher.stop() @@ -789,12 +742,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): if self._sync_enc_pool is not None: self._sync_enc_pool.close() self._sync_enc_pool = None - # close the actual database - if self._db_handle is not None: - self._db_handle.close() - self._db_handle = None - # --------------------------------------- # close the sync database if self._sync_db is not None: self._sync_db.close() @@ -805,20 +753,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): del self.sync_queue self.sync_queue = None - def __del__(self): - """ - Free resources when deleting or garbage collecting the database. - - This is only here to minimze problems if someone ever forgets to call - the close() method after using the database; you should not rely on - garbage collecting to free up the database resources. - """ - self.close() - - @property - def replica_uid(self): - return self._get_replica_uid() - # # Exceptions # @@ -831,4 +765,13 @@ class DatabaseIsNotEncrypted(Exception): pass +def soledad_doc_factory(doc_id=None, rev=None, json='{}', has_conflicts=False, + syncable=True): + """ + Return a default Soledad Document. + Used in the initialization for SQLCipherDatabase + """ + return SoledadDocument(doc_id=doc_id, rev=rev, json=json, + has_conflicts=has_conflicts, syncable=syncable) + sqlite_backend.SQLiteDatabase.register_implementation(SQLCipherDatabase) -- cgit v1.2.3 From 8f4daa13744c049dcc96eb2cb780df1e9ba08738 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 2 Oct 2014 06:17:57 -0500 Subject: Separate soledad interfaces * Separate local storage, syncers and shared_db * Comment out unused need_sync method * Use twisted LoopingCall * Create a threadpool for syncs * Return deferred from sync method * Do not pass crypto to SQLCipherDatabase * Pass replica_uid to u1db_syncer * Rename / reorganize some initialization methods --- client/src/leap/soledad/client/adbapi.py | 28 +- client/src/leap/soledad/client/api.py | 613 ++++++--------------- client/src/leap/soledad/client/examples/use_api.py | 2 +- client/src/leap/soledad/client/interfaces.py | 361 ++++++++++++ client/src/leap/soledad/client/pragmas.py | 13 +- client/src/leap/soledad/client/secrets.py | 15 +- client/src/leap/soledad/client/sqlcipher.py | 326 ++++++----- client/src/leap/soledad/client/sync.py | 7 +- client/src/leap/soledad/client/target.py | 45 +- common/src/leap/soledad/common/tests/util.py | 3 +- 10 files changed, 757 insertions(+), 656 deletions(-) create mode 100644 client/src/leap/soledad/client/interfaces.py diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index 3b15509b..60d9e195 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -30,7 +30,7 @@ from u1db.backends import sqlite_backend from twisted.enterprise import adbapi from twisted.python import log -from leap.soledad.client.sqlcipher import set_init_pragmas +from leap.soledad.client import sqlcipher as soledad_sqlcipher DEBUG_SQL = os.environ.get("LEAP_DEBUG_SQL") @@ -40,18 +40,15 @@ if DEBUG_SQL: def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): if openfun is None and driver == "pysqlcipher": - openfun = partial(set_init_pragmas, opts=opts) + openfun = partial(soledad_sqlcipher.set_init_pragmas, opts=opts) return U1DBConnectionPool( "%s.dbapi2" % driver, database=opts.path, check_same_thread=False, cp_openfun=openfun) -# XXX work in progress -------------------------------------------- - - -class U1DBSqliteWrapper(sqlite_backend.SQLitePartialExpandDatabase): +class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase): """ - A very simple wrapper around sqlcipher backend. + A very simple wrapper for u1db around sqlcipher backend. Instead of initializing the database on the fly, it just uses an existing connection that is passed to it in the initializer. @@ -64,9 +61,24 @@ class U1DBSqliteWrapper(sqlite_backend.SQLitePartialExpandDatabase): self._factory = u1db.Document +class SoledadSQLCipherWrapper(soledad_sqlcipher.SQLCipherDatabase): + """ + A wrapper for u1db that uses the Soledad-extended sqlcipher backend. + + Instead of initializing the database on the fly, it just uses an existing + connection that is passed to it in the initializer. + """ + def __init__(self, conn): + self._db_handle = conn + self._real_replica_uid = None + self._ensure_schema() + self.set_document_factory(soledad_sqlcipher.soledad_doc_factory) + self._prime_replica_uid() + + class U1DBConnection(adbapi.Connection): - u1db_wrapper = U1DBSqliteWrapper + u1db_wrapper = SoledadSQLCipherWrapper def __init__(self, pool, init_u1db=False): self.init_u1db = init_u1db diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 703b9516..493f6c1d 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -41,6 +41,9 @@ except ImportError: from u1db.remote import http_client from u1db.remote.ssl_match_hostname import match_hostname +from zope.interface import implements + +from twisted.python import log from leap.common.config import get_path_prefix from leap.soledad.common import SHARED_DB_NAME @@ -49,11 +52,11 @@ from leap.soledad.common import soledad_assert_type from leap.soledad.client import adbapi from leap.soledad.client import events as soledad_events +from leap.soledad.client import interfaces as soledad_interfaces from leap.soledad.client.crypto import SoledadCrypto from leap.soledad.client.secrets import SoledadSecrets from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client.target import SoledadSyncTarget -from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client.sqlcipher import SQLCipherOptions, SQLCipherU1DBSync logger = logging.getLogger(name=__name__) @@ -61,17 +64,13 @@ logger = logging.getLogger(name=__name__) # Constants # -SOLEDAD_CERT = None """ Path to the certificate file used to certify the SSL connection between Soledad client and server. """ +SOLEDAD_CERT = None -# -# Soledad: local encrypted storage and remote encrypted sync. -# - class Soledad(object): """ Soledad provides encrypted data storage and sync. @@ -104,65 +103,57 @@ class Soledad(object): SOLEDAD_DONE_DATA_SYNC: emitted inside C{sync()} method when it has finished synchronizing with remote replica. """ + implements(soledad_interfaces.ILocalStorage, + soledad_interfaces.ISyncableStorage, + soledad_interfaces.ISharedSecretsStorage) - LOCAL_DATABASE_FILE_NAME = 'soledad.u1db' - """ - The name of the local SQLCipher U1DB database file. - """ - - STORAGE_SECRETS_FILE_NAME = "soledad.json" - """ - The name of the file where the storage secrets will be stored. - """ - - DEFAULT_PREFIX = os.path.join(get_path_prefix(), 'leap', 'soledad') - """ - Prefix for default values for path. - """ + local_db_file_name = 'soledad.u1db' + secrets_file_name = "soledad.json" + default_prefix = os.path.join(get_path_prefix(), 'leap', 'soledad') def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, - auth_token=None, secret_id=None, defer_encryption=False): + auth_token=None, defer_encryption=False): """ Initialize configuration, cryptographic keys and dbs. :param uuid: User's uuid. :type uuid: str - :param passphrase: The passphrase for locking and unlocking encryption - secrets for local and remote storage. + :param passphrase: + The passphrase for locking and unlocking encryption secrets for + local and remote storage. :type passphrase: unicode - :param secrets_path: Path for storing encrypted key used for - symmetric encryption. + :param secrets_path: + Path for storing encrypted key used for symmetric encryption. :type secrets_path: str :param local_db_path: Path for local encrypted storage db. :type local_db_path: str - :param server_url: URL for Soledad server. This is used either to sync - with the user's remote db and to interact with the - shared recovery database. + :param server_url: + URL for Soledad server. This is used either to sync with the user's + remote db and to interact with the shared recovery database. :type server_url: str - :param cert_file: Path to the certificate of the ca used - to validate the SSL certificate used by the remote - soledad server. + :param cert_file: + Path to the certificate of the ca used to validate the SSL + certificate used by the remote soledad server. :type cert_file: str - :param auth_token: Authorization token for accessing remote databases. + :param auth_token: + Authorization token for accessing remote databases. :type auth_token: str - :param secret_id: The id of the storage secret to be used. - :type secret_id: str - - :param defer_encryption: Whether to defer encryption/decryption of - documents, or do it inline while syncing. + :param defer_encryption: + Whether to defer encryption/decryption of documents, or do it + inline while syncing. :type defer_encryption: bool - :raise BootstrapSequenceError: Raised when the secret generation and - storage on server sequence has failed - for some reason. + :raise BootstrapSequenceError: + Raised when the secret generation and storage on server sequence + has failed for some reason. """ # store config params self._uuid = uuid @@ -170,30 +161,34 @@ class Soledad(object): self._secrets_path = secrets_path self._local_db_path = local_db_path self._server_url = server_url + self._defer_encryption = defer_encryption + + self.shared_db = None + # configure SSL certificate global SOLEDAD_CERT SOLEDAD_CERT = cert_file - self._set_token(auth_token) - self._defer_encryption = defer_encryption - - self._init_config() - self._init_dirs() # init crypto variables - self._shared_db_instance = None + self._set_token(auth_token) self._crypto = SoledadCrypto(self) - self._secrets = SoledadSecrets( - self._uuid, - self._passphrase, - self._secrets_path, - self._shared_db, - self._crypto, - secret_id=secret_id) - # initiate bootstrap sequence - self._bootstrap() # might raise BootstrapSequenceError() + self._init_config_with_defaults() + self._init_working_dirs() + + # Initialize shared recovery database + self.init_shared_db(server_url, uuid, self._creds) - def _init_config(self): + # The following can raise BootstrapSequenceError, that will be + # propagated upwards. + self._init_secrets() + self._init_u1db_sqlcipher_backend() + self._init_u1db_syncer() + + # + # initialization/destruction methods + # + def _init_config_with_defaults(self): """ Initialize configuration using default values for missing params. """ @@ -202,55 +197,37 @@ class Soledad(object): # initialize secrets_path initialize(self._secrets_path, os.path.join( - self.DEFAULT_PREFIX, self.STORAGE_SECRETS_FILE_NAME)) - + self.default_prefix, self.secrets_file_name)) # initialize local_db_path initialize(self._local_db_path, os.path.join( - self.DEFAULT_PREFIX, self.LOCAL_DATABASE_FILE_NAME)) - + self.default_prefix, self.local_db_file_name)) # initialize server_url soledad_assert(self._server_url is not None, 'Missing URL for Soledad server.') - # - # initialization/destruction methods - # - - def _bootstrap(self): - """ - Bootstrap local Soledad instance. - - :raise BootstrapSequenceError: - Raised when the secret generation and storage on server sequence - has failed for some reason. - """ - self._secrets.bootstrap() - self._init_db() - # XXX initialize syncers? - - def _init_dirs(self): + def _init_working_dirs(self): """ Create work directories. :raise OSError: in case file exists and is not a dir. """ - paths = map( - lambda x: os.path.dirname(x), - [self._local_db_path, self._secrets_path]) + paths = map(lambda x: os.path.dirname(x), [ + self._local_db_path, self._secrets_path]) for path in paths: - try: - if not os.path.isdir(path): - logger.info('Creating directory: %s.' % path) - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - def _init_db(self): + create_path_if_not_exists(path) + + def _init_secrets(self): + self._secrets = SoledadSecrets( + self.uuid, self.passphrase, self.secrets_path, + self._shared_db, self._crypto) + self._secrets.bootstrap() + + def _init_u1db_sqlcipher_backend(self): """ - Initialize the U1DB SQLCipher database for local storage. + Initialize the U1DB SQLCipher database for local storage, by + instantiating a modified twisted adbapi that will maintain a threadpool + with a u1db-sqclipher connection for each thread, and will return + deferreds for each u1db query. Currently, Soledad uses the default SQLCipher cipher, i.e. 'aes-256-cbc'. We use scrypt to derive a 256-bit encryption key, @@ -268,8 +245,17 @@ class Soledad(object): defer_encryption=self._defer_encryption, sync_db_key=sync_db_key, ) + self._soledad_opts = opts self._dbpool = adbapi.getConnectionPool(opts) + def _init_u1db_syncer(self): + self._dbsyncer = SQLCipherU1DBSync( + self._soledad_opts, self._crypto, self._defer_encryption) + + # + # Closing methods + # + def close(self): """ Close underlying U1DB database. @@ -279,401 +265,164 @@ class Soledad(object): # TODO close syncers >>>>>> - #if hasattr(self, '_db') and isinstance( - #self._db, - #SQLCipherDatabase): - #self._db.close() -# - # XXX stop syncers - # self._db.stop_sync() - - @property - def _shared_db(self): - """ - Return an instance of the shared recovery database object. - - :return: The shared database. - :rtype: SoledadSharedDatabase - """ - if self._shared_db_instance is None: - self._shared_db_instance = SoledadSharedDatabase.open_database( - urlparse.urljoin(self.server_url, SHARED_DB_NAME), - self._uuid, - False, # db should exist at this point. - creds=self._creds) - return self._shared_db_instance - # - # Document storage, retrieval and sync. + # ILocalStorage # def put_doc(self, doc): - # TODO what happens with this warning during the deferred life cycle? - # Isn't it better to defend ourselves from the mutability, to avoid - # nasty surprises? """ - Update a document in the local encrypted database. - ============================== WARNING ============================== This method converts the document's contents to unicode in-place. This means that after calling `put_doc(doc)`, the contents of the document, i.e. `doc.content`, might be different from before the call. ============================== WARNING ============================== - - :param doc: the document to update - :type doc: SoledadDocument - - :return: - a deferred that will fire with the new revision identifier for - the document - :rtype: Deferred """ + # TODO what happens with this warning during the deferred life cycle? + # Isn't it better to defend ourselves from the mutability, to avoid + # nasty surprises? doc.content = self._convert_to_unicode(doc.content) return self._dbpool.put_doc(doc) def delete_doc(self, doc): - """ - Delete a document from the local encrypted database. - - :param doc: the document to delete - :type doc: SoledadDocument - - :return: - a deferred that will fire with ... - :rtype: Deferred - """ # XXX what does this do when fired??? return self._dbpool.delete_doc(doc) def get_doc(self, doc_id, include_deleted=False): - """ - Retrieve a document from the local encrypted database. - - :param doc_id: the unique document identifier - :type doc_id: str - :param include_deleted: - if True, deleted documents will be returned with empty content; - otherwise asking for a deleted document will return None - :type include_deleted: bool - - :return: - A deferred that will fire with the document object, containing a - SoledadDocument, or None if it could not be found - :rtype: Deferred - """ return self._dbpool.get_doc(doc_id, include_deleted=include_deleted) def get_docs(self, doc_ids, check_for_conflicts=True, include_deleted=False): - """ - Get the content for many documents. - - :param doc_ids: a list of document identifiers - :type doc_ids: list - :param check_for_conflicts: if set False, then the conflict check will - be skipped, and 'None' will be returned instead of True/False - :type check_for_conflicts: bool - - :return: - A deferred that will fire with an iterable giving the Document - object for each document id in matching doc_ids order. - :rtype: Deferred - """ - return self._dbpool.get_docs( - doc_ids, check_for_conflicts=check_for_conflicts, - include_deleted=include_deleted) + return self._dbpool.get_docs(doc_ids, + check_for_conflicts=check_for_conflicts, + include_deleted=include_deleted) def get_all_docs(self, include_deleted=False): - """ - Get the JSON content for all documents in the database. - - :param include_deleted: If set to True, deleted documents will be - returned with empty content. Otherwise deleted - documents will not be included in the results. - :return: - A deferred that will fire with (generation, [Document]): that is, - the current generation of the database, followed by a list of all - the documents in the database. - :rtype: Deferred - """ return self._dbpool.get_all_docs(include_deleted) def create_doc(self, content, doc_id=None): - """ - Create a new document in the local encrypted database. - - :param content: the contents of the new document - :type content: dict - :param doc_id: an optional identifier specifying the document id - :type doc_id: str - - :return: - A deferred tht will fire with the new document (SoledadDocument - instance). - :rtype: Deferred - """ return self._dbpool.create_doc( _convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): - """ - Create a new document. - - You can optionally specify the document identifier, but the document - must not already exist. See 'put_doc' if you want to override an - existing document. - If the database specifies a maximum document size and the document - exceeds it, create will fail and raise a DocumentTooBig exception. - - :param json: The JSON document string - :type json: str - :param doc_id: An optional identifier specifying the document id. - :type doc_id: - :return: - A deferred that will fire with the new document (A SoledadDocument - instance) - :rtype: Deferred - """ return self._dbpool.create_doc_from_json(json, doc_id=doc_id) def create_index(self, index_name, *index_expressions): - """ - Create an named index, which can then be queried for future lookups. - Creating an index which already exists is not an error, and is cheap. - Creating an index which does not match the index_expressions of the - existing index is an error. - Creating an index will block until the expressions have been evaluated - and the index generated. - - :param index_name: A unique name which can be used as a key prefix - :type index_name: str - :param index_expressions: - index expressions defining the index information. - :type index_expressions: dict - - Examples: - - "fieldname", or "fieldname.subfieldname" to index alphabetically - sorted on the contents of a field. - - "number(fieldname, width)", "lower(fieldname)" - """ return self._dbpool.create_index(index_name, *index_expressions) def delete_index(self, index_name): - """ - Remove a named index. - - :param index_name: The name of the index we are removing - :type index_name: str - """ return self._dbpool.delete_index(index_name) def list_indexes(self): - """ - List the definitions of all known indexes. - - :return: A list of [('index-name', ['field', 'field2'])] definitions. - :rtype: list - """ return self._dbpool.list_indexes() def get_from_index(self, index_name, *key_values): - """ - Return documents that match the keys supplied. - - You must supply exactly the same number of values as have been defined - in the index. It is possible to do a prefix match by using '*' to - indicate a wildcard match. You can only supply '*' to trailing entries, - (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) - It is also possible to append a '*' to the last supplied value (eg - 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - - :param index_name: The index to query - :type index_name: str - :param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - :type key_values: tuple - :return: List of [Document] - :rtype: list - """ return self._dbpool.get_from_index(index_name, *key_values) def get_count_from_index(self, index_name, *key_values): - """ - Return the count of the documents that match the keys and - values supplied. - - :param index_name: The index to query - :type index_name: str - :param key_values: values to match. eg, if you have - an index with 3 fields then you would have: - get_from_index(index_name, val1, val2, val3) - :type key_values: tuple - :return: count. - :rtype: int - """ return self._dbpool.get_count_from_index(index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): - """ - Return documents that fall within the specified range. - - Both ends of the range are inclusive. For both start_value and - end_value, one must supply exactly the same number of values as have - been defined in the index, or pass None. In case of a single column - index, a string is accepted as an alternative for a tuple with a single - value. It is possible to do a prefix match by using '*' to indicate - a wildcard match. You can only supply '*' to trailing entries, (eg - 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also - possible to append a '*' to the last supplied value (eg 'val*', '*', - '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') - - :param index_name: The index to query - :type index_name: str - :param start_values: tuples of values that define the lower bound of - the range. eg, if you have an index with 3 fields then you would - have: (val1, val2, val3) - :type start_values: tuple - :param end_values: tuples of values that define the upper bound of the - range. eg, if you have an index with 3 fields then you would have: - (val1, val2, val3) - :type end_values: tuple - :return: A deferred that will fire with a list of [Document] - :rtype: Deferred - """ return self._dbpool.get_range_from_index( index_name, start_value, end_value) def get_index_keys(self, index_name): - """ - Return all keys under which documents are indexed in this index. - - :param index_name: The index to query - :type index_name: str - :return: - A deferred that will fire with a list of tuples of indexed keys. - :rtype: Deferred - """ return self._dbpool.get_index_keys(index_name) def get_doc_conflicts(self, doc_id): - """ - Get the list of conflicts for the given document. - - :param doc_id: the document id - :type doc_id: str - - :return: - A deferred that will fire with a list of the document entries that - are conflicted. - :rtype: Deferred - """ return self._dbpool.get_doc_conflicts(doc_id) def resolve_doc(self, doc, conflicted_doc_revs): - """ - Mark a document as no longer conflicted. - - :param doc: a document with the new content to be inserted. - :type doc: SoledadDocument - :param conflicted_doc_revs: - A deferred that will fire with a list of revisions that the new - content supersedes. - :type conflicted_doc_revs: list - """ return self._dbpool.resolve_doc(doc, conflicted_doc_revs) + def _get_local_db_path(self): + return self._local_db_path + + # XXX Do we really need all this private / property dance? + + local_db_path = property( + _get_local_db_path, + doc='The path for the local database replica.') + + def _get_uuid(self): + return self._uuid + + uuid = property(_get_uuid, doc='The user uuid.') + # - # Sync API + # ISyncableStorage # - # TODO have interfaces, and let it implement it. - def sync(self, defer_decryption=True): - """ - Synchronize the local encrypted replica with a remote replica. - This method blocks until a syncing lock is acquired, so there are no - attempts of concurrent syncs from the same client replica. + # ----------------------------------------------------------------- + # TODO this needs work. + # Should review/write tests to check that this: - :param url: the url of the target replica to sync with - :type url: str + # (1) Defer to the syncer pool -- DONE (on dbsyncer) + # (2) Return the deferred + # (3) Add the callback for signaling the event (executed on reactor + # thread) + # (4) Check that the deferred is called with the local gen. - :param defer_decryption: - Whether to defer the decryption process using the intermediate - database. If False, decryption will be done inline. - :type defer_decryption: bool + # TODO document that this returns a deferred + # ----------------------------------------------------------------- - :return: - A deferred that will fire with the local generation before the - synchronisation was performed. - :rtype: str - """ - # TODO this needs work. - # Should: - # (1) Defer to the syncer pool - # (2) Return a deferred (the deferToThreadpool can be good) - # (3) Add the callback for signaling the event - # (4) Let the local gen be returned from the thread + def on_sync_done(local_gen): + soledad_events.signal( + soledad_events.SOLEDAD_DONE_DATA_SYNC, self.uuid) + return local_gen + + sync_url = urlparse.urljoin(self.server_url, 'user-%s' % self.uuid) try: - local_gen = self._dbsyncer.sync( - urlparse.urljoin(self.server_url, 'user-%s' % self._uuid), + d = self._dbsyncer.sync( + sync_url, creds=self._creds, autocreate=False, defer_decryption=defer_decryption) - soledad_events.signal( - soledad_events.SOLEDAD_DONE_DATA_SYNC, self._uuid) - return local_gen + + d.addCallbacks(on_sync_done, lambda err: log.err(err)) + return d + + # TODO catch the exception by adding an Errback except Exception as e: logger.error("Soledad exception when syncing: %s" % str(e)) def stop_sync(self): - """ - Stop the current syncing process. - """ self._dbsyncer.stop_sync() - def need_sync(self, url): - """ - Return if local db replica differs from remote url's replica. - - :param url: The remote replica to compare with local replica. - :type url: str - - :return: Whether remote replica and local replica differ. - :rtype: bool - """ - # XXX pass the get_replica_uid ------------------------ - # From where? initialize with that? - replica_uid = self._db._get_replica_uid() - target = SoledadSyncTarget( - url, replica_uid, creds=self._creds, crypto=self._crypto) - - generation = self._db._get_generation() + # FIXME ------------------------------------------------------- + # review if we really need this. I think that we can the sync + # fail silently if nothing is to be synced. + #def need_sync(self, url): + # XXX dispatch this method in the dbpool ................. + #replica_uid = self._dbpool.replica_uid + #target = SoledadSyncTarget( + #url, replica_uid, creds=self._creds, crypto=self._crypto) +# + # XXX does it matter if we get this from the general dbpool or the + # syncer pool? + #generation = self._dbpool.get_generation() +# # XXX better unpack it? - info = target.get_sync_info(replica_uid) - + #info = target.get_sync_info(replica_uid) +# # compare source generation with target's last known source generation - if generation != info[4]: - soledad_events.signal( - soledad_events.SOLEDAD_NEW_DATA_TO_SYNC, self._uuid) - return True - return False + #if generation != info[4]: + #soledad_events.signal( + #soledad_events.SOLEDAD_NEW_DATA_TO_SYNC, self.uuid) + #return True + #return False @property def syncing(self): - """ - Property, True if the syncer is syncing. - """ return self._dbsyncer.syncing def _set_token(self, token): """ Set the authentication token for remote database access. - Build the credentials dictionary with the following format: + Internally, this builds the credentials dictionary with the following + format: self._{ 'token': { @@ -686,7 +435,7 @@ class Soledad(object): """ self._creds = { 'token': { - 'uuid': self._uuid, + 'uuid': self.uuid, 'token': token, } } @@ -699,25 +448,24 @@ class Soledad(object): token = property(_get_token, _set_token, doc='The authentication Token.') - # - # Setters/getters - # - - def _get_uuid(self): - return self._uuid - - uuid = property(_get_uuid, doc='The user uuid.') + def _get_server_url(self): + return self._server_url - def get_secret_id(self): - return self._secrets.secret_id + server_url = property( + _get_server_url, + doc='The URL of the Soledad server.') - def set_secret_id(self, secret_id): - self._secrets.set_secret_id(secret_id) + # + # ISharedSecretsStorage + # - secret_id = property( - get_secret_id, - set_secret_id, - doc='The active secret id.') + def init_shared_db(self, server_url, uuid, creds): + shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) + self.shared_db = SoledadSharedDatabase.open_database( + shared_db_url, + uuid, + creds=creds, + create=False) # db should exist at this point. def _set_secrets_path(self, secrets_path): self._secrets.secrets_path = secrets_path @@ -730,20 +478,6 @@ class Soledad(object): _set_secrets_path, doc='The path for the file containing the encrypted symmetric secret.') - def _get_local_db_path(self): - return self._local_db_path - - local_db_path = property( - _get_local_db_path, - doc='The path for the local database replica.') - - def _get_server_url(self): - return self._server_url - - server_url = property( - _get_server_url, - doc='The URL of the Soledad server.') - @property def storage_secret(self): """ @@ -762,19 +496,7 @@ class Soledad(object): def secrets(self): return self._secrets - @property - def passphrase(self): - return self._secrets.passphrase - def change_passphrase(self, new_passphrase): - """ - Change the passphrase that encrypts the storage secret. - - :param new_passphrase: The new passphrase. - :type new_passphrase: unicode - - :raise NoStorageSecret: Raised if there's no storage secret available. - """ self._secrets.change_passphrase(new_passphrase) @@ -811,6 +533,17 @@ def _convert_to_unicode(content): return content +def create_path_if_not_exists(path): + try: + if not os.path.isdir(path): + logger.info('Creating directory: %s.' % path) + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + # ---------------------------------------------------------------------------- # Monkey patching u1db to be able to provide a custom SSL cert # ---------------------------------------------------------------------------- diff --git a/client/src/leap/soledad/client/examples/use_api.py b/client/src/leap/soledad/client/examples/use_api.py index fd0a100c..4268fe71 100644 --- a/client/src/leap/soledad/client/examples/use_api.py +++ b/client/src/leap/soledad/client/examples/use_api.py @@ -46,7 +46,7 @@ if os.path.isfile(tmpdb): start_time = datetime.datetime.now() opts = SQLCipherOptions(tmpdb, "secret", create=True) -db = sqlcipher.SQLCipherDatabase(None, opts) +db = sqlcipher.SQLCipherDatabase(opts) def allDone(): diff --git a/client/src/leap/soledad/client/interfaces.py b/client/src/leap/soledad/client/interfaces.py new file mode 100644 index 00000000..6bd3f200 --- /dev/null +++ b/client/src/leap/soledad/client/interfaces.py @@ -0,0 +1,361 @@ +# -*- coding: utf-8 -*- +# interfaces.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Interfaces used by the Soledad Client. +""" +from zope.interface import Interface, Attribute + + +class ILocalStorage(Interface): + """ + I implement core methods for the u1db local storage. + """ + local_db_path = Attribute( + "The path for the local database replica") + local_db_file_name = Attribute( + "The name of the local SQLCipher U1DB database file") + uuid = Attribute("The user uuid") + default_prefix = Attribute( + "Prefix for default values for path") + + def put_doc(self, doc): + """ + Update a document in the local encrypted database. + + :param doc: the document to update + :type doc: SoledadDocument + + :return: + a deferred that will fire with the new revision identifier for + the document + :rtype: Deferred + """ + + def delete_doc(self, doc): + """ + Delete a document from the local encrypted database. + + :param doc: the document to delete + :type doc: SoledadDocument + + :return: + a deferred that will fire with ... + :rtype: Deferred + """ + + def get_doc(self, doc_id, include_deleted=False): + """ + Retrieve a document from the local encrypted database. + + :param doc_id: the unique document identifier + :type doc_id: str + :param include_deleted: + if True, deleted documents will be returned with empty content; + otherwise asking for a deleted document will return None + :type include_deleted: bool + + :return: + A deferred that will fire with the document object, containing a + SoledadDocument, or None if it could not be found + :rtype: Deferred + """ + + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + """ + Get the content for many documents. + + :param doc_ids: a list of document identifiers + :type doc_ids: list + :param check_for_conflicts: if set False, then the conflict check will + be skipped, and 'None' will be returned instead of True/False + :type check_for_conflicts: bool + + :return: + A deferred that will fire with an iterable giving the Document + object for each document id in matching doc_ids order. + :rtype: Deferred + """ + + def get_all_docs(self, include_deleted=False): + """ + Get the JSON content for all documents in the database. + + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted + documents will not be included in the results. + :return: + A deferred that will fire with (generation, [Document]): that is, + the current generation of the database, followed by a list of all + the documents in the database. + :rtype: Deferred + """ + + def create_doc(self, content, doc_id=None): + """ + Create a new document in the local encrypted database. + + :param content: the contents of the new document + :type content: dict + :param doc_id: an optional identifier specifying the document id + :type doc_id: str + + :return: + A deferred tht will fire with the new document (SoledadDocument + instance). + :rtype: Deferred + """ + + def create_doc_from_json(self, json, doc_id=None): + """ + Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param json: The JSON document string + :type json: str + :param doc_id: An optional identifier specifying the document id. + :type doc_id: + :return: + A deferred that will fire with the new document (A SoledadDocument + instance) + :rtype: Deferred + """ + + def create_index(self, index_name, *index_expressions): + """ + Create an named index, which can then be queried for future lookups. + Creating an index which already exists is not an error, and is cheap. + Creating an index which does not match the index_expressions of the + existing index is an error. + Creating an index will block until the expressions have been evaluated + and the index generated. + + :param index_name: A unique name which can be used as a key prefix + :type index_name: str + :param index_expressions: + index expressions defining the index information. + :type index_expressions: dict + + Examples: + + "fieldname", or "fieldname.subfieldname" to index alphabetically + sorted on the contents of a field. + + "number(fieldname, width)", "lower(fieldname)" + """ + + def delete_index(self, index_name): + """ + Remove a named index. + + :param index_name: The name of the index we are removing + :type index_name: str + """ + + def list_indexes(self): + """ + List the definitions of all known indexes. + + :return: A list of [('index-name', ['field', 'field2'])] definitions. + :rtype: Deferred + """ + + def get_from_index(self, index_name, *key_values): + """ + Return documents that match the keys supplied. + + You must supply exactly the same number of values as have been defined + in the index. It is possible to do a prefix match by using '*' to + indicate a wildcard match. You can only supply '*' to trailing entries, + (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) + It is also possible to append a '*' to the last supplied value (eg + 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: List of [Document] + :rtype: list + """ + + def get_count_from_index(self, index_name, *key_values): + """ + Return the count of the documents that match the keys and + values supplied. + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: count. + :rtype: int + """ + + def get_range_from_index(self, index_name, start_value, end_value): + """ + Return documents that fall within the specified range. + + Both ends of the range are inclusive. For both start_value and + end_value, one must supply exactly the same number of values as have + been defined in the index, or pass None. In case of a single column + index, a string is accepted as an alternative for a tuple with a single + value. It is possible to do a prefix match by using '*' to indicate + a wildcard match. You can only supply '*' to trailing entries, (eg + 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also + possible to append a '*' to the last supplied value (eg 'val*', '*', + '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param start_values: tuples of values that define the lower bound of + the range. eg, if you have an index with 3 fields then you would + have: (val1, val2, val3) + :type start_values: tuple + :param end_values: tuples of values that define the upper bound of the + range. eg, if you have an index with 3 fields then you would have: + (val1, val2, val3) + :type end_values: tuple + :return: A deferred that will fire with a list of [Document] + :rtype: Deferred + """ + + def get_index_keys(self, index_name): + """ + Return all keys under which documents are indexed in this index. + + :param index_name: The index to query + :type index_name: str + :return: + A deferred that will fire with a list of tuples of indexed keys. + :rtype: Deferred + """ + + def get_doc_conflicts(self, doc_id): + """ + Get the list of conflicts for the given document. + + :param doc_id: the document id + :type doc_id: str + + :return: + A deferred that will fire with a list of the document entries that + are conflicted. + :rtype: Deferred + """ + + def resolve_doc(self, doc, conflicted_doc_revs): + """ + Mark a document as no longer conflicted. + + :param doc: a document with the new content to be inserted. + :type doc: SoledadDocument + :param conflicted_doc_revs: + A deferred that will fire with a list of revisions that the new + content supersedes. + :type conflicted_doc_revs: list + """ + + +class ISyncableStorage(Interface): + """ + I implement methods to synchronize with a remote replica. + """ + replica_uid = Attribute("The uid of the local replica") + server_url = Attribute("The URL of the Soledad server.") + syncing = Attribute( + "Property, True if the syncer is syncing.") + token = Attribute("The authentication Token.") + + def sync(self, defer_decryption=True): + """ + Synchronize the local encrypted replica with a remote replica. + + This method blocks until a syncing lock is acquired, so there are no + attempts of concurrent syncs from the same client replica. + + :param url: the url of the target replica to sync with + :type url: str + + :param defer_decryption: + Whether to defer the decryption process using the intermediate + database. If False, decryption will be done inline. + :type defer_decryption: bool + + :return: + A deferred that will fire with the local generation before the + synchronisation was performed. + :rtype: str + """ + + def stop_sync(self): + """ + Stop the current syncing process. + """ + + +class ISharedSecretsStorage(Interface): + """ + I implement methods needed for the Shared Recovery Database. + """ + secrets_path = Attribute( + "Path for storing encrypted key used for symmetric encryption.") + secrets_file_name = Attribute( + "The name of the file where the storage secrets will be stored") + + storage_secret = Attribute("") + remote_storage_secret = Attribute("") + shared_db = Attribute("The shared db object") + + # XXX this used internally from secrets, so it might be good to preserve + # as a public boundary with other components. + secrets = Attribute("") + + def init_shared_db(self, server_url, uuid, creds): + """ + Initialize the shared recovery database. + + :param server_url: + :type server_url: + :param uuid: + :type uuid: + :param creds: + :type creds: + """ + + def change_passphrase(self, new_passphrase): + """ + Change the passphrase that encrypts the storage secret. + + :param new_passphrase: The new passphrase. + :type new_passphrase: unicode + + :raise NoStorageSecret: Raised if there's no storage secret available. + """ + + # XXX not in use. Uncomment if we ever decide to allow + # multiple secrets. + # secret_id = Attribute("The id of the storage secret to be used") diff --git a/client/src/leap/soledad/client/pragmas.py b/client/src/leap/soledad/client/pragmas.py index 7a13a694..2e9c53a3 100644 --- a/client/src/leap/soledad/client/pragmas.py +++ b/client/src/leap/soledad/client/pragmas.py @@ -43,7 +43,7 @@ def set_crypto_pragmas(db_handle, sqlcipher_opts): def _set_key(db_handle, key, is_raw_key): """ - Set the C{key} for use with the database. + Set the ``key`` for use with the database. The process of creating a new, encrypted database is called 'keying' the database. SQLCipher uses just-in-time key derivation at the point @@ -60,9 +60,9 @@ def _set_key(db_handle, key, is_raw_key): :param key: The key for use with the database. :type key: str - :param is_raw_key: Whether C{key} is a raw 64-char hex string or a - passphrase that should be hashed to obtain the - encyrption key. + :param is_raw_key: + Whether C{key} is a raw 64-char hex string or a passphrase that should + be hashed to obtain the encyrption key. :type is_raw_key: bool """ if is_raw_key: @@ -321,14 +321,11 @@ def set_write_ahead_logging(db_handle): """ logger.debug("SQLCIPHER: SETTING WRITE-AHEAD LOGGING") db_handle.cursor().execute('PRAGMA journal_mode=WAL') + # The optimum value can still use a little bit of tuning, but we favor # small sizes of the WAL file to get fast reads, since we assume that # the writes will be quick enough to not block too much. - # TODO - # As a further improvement, we might want to set autocheckpoint to 0 - # here and do the checkpoints manually in a separate thread, to avoid - # any blocks in the main thread (we should run a loopingcall from here) db_handle.cursor().execute('PRAGMA wal_autocheckpoint=50') diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 970ac82f..93f8c25d 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -144,8 +144,7 @@ class SoledadSecrets(object): Keys used to access storage secrets in recovery documents. """ - def __init__(self, uuid, passphrase, secrets_path, shared_db, crypto, - secret_id=None): + def __init__(self, uuid, passphrase, secrets_path, shared_db, crypto): """ Initialize the secrets manager. @@ -161,17 +160,20 @@ class SoledadSecrets(object): :type shared_db: leap.soledad.client.shared_db.SoledadSharedDatabase :param crypto: A soledad crypto object. :type crypto: SoledadCrypto - :param secret_id: The id of the storage secret to be used. - :type secret_id: str """ + # XXX removed since not in use + # We will pick the first secret available. + # param secret_id: The id of the storage secret to be used. + self._uuid = uuid self._passphrase = passphrase self._secrets_path = secrets_path self._shared_db = shared_db self._crypto = crypto - self._secret_id = secret_id self._secrets = {} + self._secret_id = None + def bootstrap(self): """ Bootstrap secrets. @@ -247,7 +249,8 @@ class SoledadSecrets(object): try: self._load_secrets() # try to load from disk except IOError as e: - logger.warning('IOError while loading secrets from disk: %s' % str(e)) + logger.warning( + 'IOError while loading secrets from disk: %s' % str(e)) return False return self.storage_secret is not None diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index c9e69c73..a7e9e0fe 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -45,7 +45,6 @@ import logging import multiprocessing import os import threading -# import time --- needed for the win initialization hack import json from hashlib import sha256 @@ -56,7 +55,10 @@ from httplib import CannotSendRequest from pysqlcipher import dbapi2 as sqlcipher_dbapi2 from u1db.backends import sqlite_backend from u1db import errors as u1db_errors -from taskthread import TimerTask + +from twisted.internet.task import LoopingCall +from twisted.internet.threads import deferToThreadPool +from twisted.python.threadpool import ThreadPool from leap.soledad.client import crypto from leap.soledad.client.target import SoledadSyncTarget @@ -64,7 +66,6 @@ from leap.soledad.client.target import PendingReceivedDocsSyncError from leap.soledad.client.sync import SoledadSynchronizer # TODO use adbapi too -from leap.soledad.client.mp_safe_db_TOREMOVE import MPSafeSQLiteDB from leap.soledad.client import pragmas from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument @@ -75,16 +76,6 @@ logger = logging.getLogger(__name__) # Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2 sqlite_backend.dbapi2 = sqlcipher_dbapi2 -# It seems that, as long as we are not using old sqlite versions, serialized -# mode is enabled by default at compile time. So accessing db connections from -# different threads should be safe, as long as no attempt is made to use them -# from multiple threads with no locking. -# See https://sqlite.org/threadsafe.html -# and http://bugs.python.org/issue16509 - -# TODO this no longer needed ------------- -#SQLITE_CHECK_SAME_THREAD = False - def initialize_sqlcipher_db(opts, on_init=None): """ @@ -96,12 +87,17 @@ def initialize_sqlcipher_db(opts, on_init=None): :type on_init: tuple :return: a SQLCipher connection """ - conn = sqlcipher_dbapi2.connect( - opts.path) + # Note: There seemed to be a bug in sqlite 3.5.9 (with python2.6) + # where without re-opening the database on Windows, it + # doesn't see the transaction that was just committed + # Removing from here now, look at the pysqlite implementation if the + # bug shows up in windows. - # XXX not needed -- check - #check_same_thread=SQLITE_CHECK_SAME_THREAD) + if not os.path.isfile(opts.path) and not opts.create: + raise u1db_errors.DatabaseDoesNotExist() + conn = sqlcipher_dbapi2.connect( + opts.path) set_init_pragmas(conn, opts, extra_queries=on_init) return conn @@ -196,10 +192,11 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ defer_encryption = False - # XXX not used afaik: - # _index_storage_value = 'expand referenced encrypted' + # The attribute _index_storage_value will be used as the lookup key. + # Here we extend it with `encrypted` + _index_storage_value = 'expand referenced encrypted' - def __init__(self, soledad_crypto, opts): + def __init__(self, opts): """ Connect to an existing SQLCipher database, creating a new sqlcipher database file if needed. @@ -217,18 +214,34 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :param opts: :type opts: SQLCipherOptions """ - # TODO ------ we don't need any soledad crypto in here - # ensure the db is encrypted if the file already exists if os.path.isfile(opts.path): - self.assert_db_is_encrypted(opts) + _assert_db_is_encrypted(opts) # connect to the sqlcipher database self._db_handle = initialize_sqlcipher_db(opts) - self._real_replica_uid = None - self._ensure_schema() + # TODO --------------------------------------------------- + # Everything else in this initialization has to be factored + # out, so it can be used from U1DBSqlcipherWrapper __init__ + # too. + # --------------------------------------------------------- + + self._ensure_schema() self.set_document_factory(soledad_doc_factory) + self._prime_replica_uid() + + def _prime_replica_uid(self): + """ + In the u1db implementation, _replica_uid is a property + that returns the value in _real_replica_uid, and does + a db query if no value found. + Here we prime the replica uid during initialization so + that we don't have to wait for the query afterwards. + """ + self._real_replica_uid = None + self._get_replica_uid() + print "REPLICA UID --->", self._real_replica_uid def _extra_schema_init(self, c): """ @@ -241,7 +254,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): :param c: The cursor for querying the database. :type c: dbapi2.cursor """ - print "CALLING EXTRA SCHEMA INIT...." c.execute( 'ALTER TABLE document ' 'ADD COLUMN syncable BOOL NOT NULL DEFAULT TRUE') @@ -263,7 +275,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc) - # XXX move to API + # TODO XXX move to API XXX if self.defer_encryption: self.sync_queue.put_nowait(doc) return doc_rev @@ -272,37 +284,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # SQLCipher API methods # - # TODO this doesn't need to be an instance method - def assert_db_is_encrypted(self, opts): - """ - Assert that the sqlcipher file contains an encrypted database. - - When opening an existing database, PRAGMA key will not immediately - throw an error if the key provided is incorrect. To test that the - database can be successfully opened with the provided key, it is - necessary to perform some operation on the database (i.e. read from - it) and confirm it is success. - - The easiest way to do this is select off the sqlite_master table, - which will attempt to read the first page of the database and will - parse the schema. - - :param opts: - """ - # We try to open an encrypted database with the regular u1db - # backend should raise a DatabaseError exception. - # If the regular backend succeeds, then we need to stop because - # the database was not properly initialized. - try: - sqlite_backend.SQLitePartialExpandDatabase(opts.path) - except sqlcipher_dbapi2.DatabaseError: - # assert that we can access it using SQLCipher with the given - # key - dummy_query = ('SELECT count(*) FROM sqlite_master',) - initialize_sqlcipher_db(opts, on_init=dummy_query) - else: - raise DatabaseIsNotEncrypted() - # Extra query methods: extensions to the base u1db sqlite implmentation. def get_count_from_index(self, index_name, *key_values): @@ -420,65 +401,10 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ self.close() - # TODO ---- rescue the fix for the windows case from here... - # @classmethod - # def _open_database(cls, sqlcipher_file, password, document_factory=None, - # crypto=None, raw_key=False, cipher='aes-256-cbc', - # kdf_iter=4000, cipher_page_size=1024, - # defer_encryption=False, sync_db_key=None): - # """ - # Open a SQLCipher database. -# - # :return: The database object. - # :rtype: SQLCipherDatabase - # """ - # cls.defer_encryption = defer_encryption - # if not os.path.isfile(sqlcipher_file): - # raise u1db_errors.DatabaseDoesNotExist() -# - # tries = 2 - # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6) - # where without re-opening the database on Windows, it - # doesn't see the transaction that was just committed - # while True: - # with cls.k_lock: - # db_handle = dbapi2.connect( - # sqlcipher_file, - # check_same_thread=SQLITE_CHECK_SAME_THREAD) -# - # try: - # set cryptographic params -# - # XXX pass only a CryptoOptions object around - #pragmas.set_crypto_pragmas( - #db_handle, password, raw_key, cipher, kdf_iter, - #cipher_page_size) - #c = db_handle.cursor() - # XXX if we use it here, it should be public - #v, err = cls._which_index_storage(c) - #except Exception as exc: - #logger.warning("ERROR OPENING DATABASE!") - #logger.debug("error was: %r" % exc) - #v, err = None, exc - #finally: - #db_handle.close() - #if v is not None: - #break - # possibly another process is initializing it, wait for it to be - # done - #if tries == 0: - #raise err # go for the richest error? - #tries -= 1 - #time.sleep(cls.WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL) - #return SQLCipherDatabase._sqlite_registry[v]( - #sqlcipher_file, password, document_factory=document_factory, - #crypto=crypto, raw_key=raw_key, cipher=cipher, kdf_iter=kdf_iter, - #cipher_page_size=cipher_page_size, sync_db_key=sync_db_key) - class SQLCipherU1DBSync(object): - _sync_watcher = None + _sync_loop = None _sync_enc_pool = None """ @@ -495,11 +421,10 @@ class SQLCipherU1DBSync(object): encrypting_lock = threading.Lock() """ - Period or recurrence of the periodic encrypting task, in seconds. + Period or recurrence of the Looping Call that will do the encryption to the + syncdb (in seconds). """ - # XXX use LoopingCall. - # Just use fucking deferreds, do not waste time looping. - ENCRYPT_TASK_PERIOD = 1 + ENCRYPT_LOOP_PERIOD = 1 """ A dictionary that hold locks which avoid multiple sync attempts from the @@ -507,39 +432,62 @@ class SQLCipherU1DBSync(object): """ syncing_lock = defaultdict(threading.Lock) - def _init_sync(self, opts, soledad_crypto, defer_encryption=False): + def __init__(self, opts, soledad_crypto, replica_uid, + defer_encryption=False): self._crypto = soledad_crypto - - # TODO ----- have to decide what to do with syncer self._sync_db_key = opts.sync_db_key self._sync_db = None self._sync_db_write_lock = None self._sync_enc_pool = None self.sync_queue = None - if self.defer_encryption: - # initialize sync db - self._init_sync_db() - # initialize syncing queue encryption pool - self._sync_enc_pool = crypto.SyncEncrypterPool( - self._crypto, self._sync_db, self._sync_db_write_lock) - self._sync_watcher = TimerTask(self._encrypt_syncing_docs, - self.ENCRYPT_TASK_PERIOD) - self._sync_watcher.start() - - # TODO move to class attribute? # we store syncers in a dictionary indexed by the target URL. We also # store a hash of the auth info in case auth info expires and we need # to rebuild the syncer for that target. The final self._syncers # format is the following:: # # self._syncers = {'': ('', syncer), ...} + self._syncers = {} self._sync_db_write_lock = threading.Lock() self.sync_queue = multiprocessing.Queue() - def _init_sync_db(self, opts): + self._sync_threadpool = None + self._initialize_sync_threadpool() + + if defer_encryption: + self._initialize_sync_db() + + # initialize syncing queue encryption pool + self._sync_enc_pool = crypto.SyncEncrypterPool( + self._crypto, self._sync_db, self._sync_db_write_lock) + + # ------------------------------------------------------------------ + # From the documentation: If f returns a deferred, rescheduling + # will not take place until the deferred has fired. The result + # value is ignored. + + # TODO use this to avoid multiple sync attempts if the sync has not + # finished! + # ------------------------------------------------------------------ + + # XXX this was called sync_watcher --- trace any remnants + self._sync_loop = LoopingCall(self._encrypt_syncing_docs), + self._sync_loop.start(self.ENCRYPT_LOOP_PERIOD) + + def _initialize_sync_threadpool(self): + """ + Initialize a ThreadPool with exactly one thread, that will be used to + run all the network blocking calls for syncing on a separate thread. + + TODO this needs to be ported away from urllib and into twisted async + calls, and then we can ditch this syncing thread and reintegrate into + the main reactor. + """ + self._sync_threadpool = ThreadPool(0, 1) + + def _initialize_sync_db(self, opts): """ Initialize the Symmetrically-Encrypted document to be synced database, and the queue to communicate with subprocess workers. @@ -554,29 +502,32 @@ class SQLCipherU1DBSync(object): else: sync_db_path = ":memory:" - # XXX use initialize_sqlcipher_db here too - # TODO pass on_init queries to initialize_sqlcipher_db - self._sync_db = MPSafeSQLiteDB(sync_db_path) - pragmas.set_crypto_pragmas(self._sync_db, opts) + # --------------------------------------------------------- + # TODO use a separate adbapi for this (sqlcipher only, no u1db) + # We could control that it only has 1 or 2 threads. - # create sync tables - self._create_sync_db_tables() + opts.path = sync_db_path - def _create_sync_db_tables(self): + self._sync_db = initialize_sqlcipher_db( + opts, on_init=self._sync_db_extra_init) + # --------------------------------------------------------- + + @property + def _sync_db_extra_init(self): """ - Create tables for the local sync documents db if needed. + Queries for creating tables for the local sync documents db if needed. + They are passed as extra initialization to initialize_sqlciphjer_db + + :rtype: tuple of strings """ - # TODO use adbapi --------------------------------- + maybe_create = "CREATE TABLE IF NOT EXISTS %s (%s)" encr = crypto.SyncEncrypterPool decr = crypto.SyncDecrypterPool - sql_encr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % ( + sql_encr_table_query = (maybe_create % ( encr.TABLE_NAME, encr.FIELD_NAMES)) - sql_decr = ("CREATE TABLE IF NOT EXISTS %s (%s)" % ( + sql_decr_table_query = (maybe_create % ( decr.TABLE_NAME, decr.FIELD_NAMES)) - - with self._sync_db_write_lock: - self._sync_db.execute(sql_encr) - self._sync_db.execute(sql_decr) + return (sql_encr_table_query, sql_decr_table_query) def sync(self, url, creds=None, autocreate=True, defer_decryption=True): """ @@ -599,15 +550,24 @@ class SQLCipherU1DBSync(object): database. If False, decryption will be done inline. :type defer_decryption: bool - :return: The local generation before the synchronisation was performed. - :rtype: int + :return: + A Deferred, that will fire with the local generation (type `int`) + before the synchronisation was performed. + :rtype: deferred """ + kwargs = {'creds': creds, 'autocreate': autocreate, + 'defer_decryption': defer_decryption} + return deferToThreadPool(self._sync, url, **kwargs) + + def _sync(self, url, creds=None, autocreate=True, defer_decryption=True): res = None + # the following context manager blocks until the syncing lock can be # acquired. - if defer_decryption: - self._init_sync_db() - with self.syncer(url, creds=creds) as syncer: + # TODO review, I think this is no longer needed with a 1-thread + # threadpool. + + with self._syncer(url, creds=creds) as syncer: # XXX could mark the critical section here... try: res = syncer.sync(autocreate=autocreate, @@ -634,7 +594,7 @@ class SQLCipherU1DBSync(object): syncer.stop() @contextmanager - def syncer(self, url, creds=None): + def _syncer(self, url, creds=None): """ Accesor for synchronizer. @@ -643,13 +603,13 @@ class SQLCipherU1DBSync(object): Because of that, this method blocks until the syncing lock can be acquired. """ - with self.syncing_lock[self._get_replica_uid()]: + with self.syncing_lock[self.replica_uid]: syncer = self._get_syncer(url, creds=creds) yield syncer @property def syncing(self): - lock = self.syncing_lock[self._get_replica_uid()] + lock = self.syncing_lock[self.replica_uid] acquired_lock = lock.acquire(False) if acquired_lock is False: return True @@ -679,7 +639,7 @@ class SQLCipherU1DBSync(object): syncer = SoledadSynchronizer( self, SoledadSyncTarget(url, - self._replica_uid, + self.replica_uid, creds=creds, crypto=self._crypto, sync_db=self._sync_db, @@ -701,8 +661,11 @@ class SQLCipherU1DBSync(object): to be encrypted in the sync db. They will be read by the SoledadSyncTarget during the sync_exchange. - Called periodical from the TimerTask self._sync_watcher. + Called periodically from the LoopingCall self._sync_loop. """ + # TODO should return a deferred that would firewhen the encryption is + # done. See note on __init__ + lock = self.encrypting_lock # optional wait flag used to avoid blocking if not lock.acquire(False): @@ -720,19 +683,19 @@ class SQLCipherU1DBSync(object): finally: lock.release() - @property - def replica_uid(self): - return self._get_replica_uid() + def get_generation(self): + # FIXME + # XXX this SHOULD BE a callback + return self._get_generation() def close(self): """ Close the syncer and syncdb orderly """ - # stop the sync watcher for deferred encryption - if self._sync_watcher is not None: - self._sync_watcher.stop() - self._sync_watcher.shutdown() - self._sync_watcher = None + # stop the sync loop for deferred encryption + if self._sync_loop is not None: + self._sync_loop.stop() + self._sync_loop = None # close all open syncers for url in self._syncers: _, syncer = self._syncers[url] @@ -753,6 +716,37 @@ class SQLCipherU1DBSync(object): del self.sync_queue self.sync_queue = None + +def _assert_db_is_encrypted(opts): + """ + Assert that the sqlcipher file contains an encrypted database. + + When opening an existing database, PRAGMA key will not immediately + throw an error if the key provided is incorrect. To test that the + database can be successfully opened with the provided key, it is + necessary to perform some operation on the database (i.e. read from + it) and confirm it is success. + + The easiest way to do this is select off the sqlite_master table, + which will attempt to read the first page of the database and will + parse the schema. + + :param opts: + """ + # We try to open an encrypted database with the regular u1db + # backend should raise a DatabaseError exception. + # If the regular backend succeeds, then we need to stop because + # the database was not properly initialized. + try: + sqlite_backend.SQLitePartialExpandDatabase(opts.path) + except sqlcipher_dbapi2.DatabaseError: + # assert that we can access it using SQLCipher with the given + # key + dummy_query = ('SELECT count(*) FROM sqlite_master',) + initialize_sqlcipher_db(opts, on_init=dummy_query) + else: + raise DatabaseIsNotEncrypted() + # # Exceptions # diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index a47afbb6..aa19ddab 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -17,10 +17,9 @@ """ Soledad synchronization utilities. - Extend u1db Synchronizer with the ability to: - * Defer the update of the known replica uid until all the decryption of + * Postpone the update of the known replica uid until all the decryption of the incoming messages has been processed. * Be interrupted and recovered. @@ -48,6 +47,8 @@ class SoledadSynchronizer(Synchronizer): Also modified to allow for interrupting the synchronization process. """ + # TODO can delegate the syncing to the api object, living in the reactor + # thread, and use a simple flag. syncing_lock = Lock() def stop(self): @@ -232,6 +233,8 @@ class SoledadSynchronizer(Synchronizer): # release if something in the syncdb-decrypt goes wrong. we could keep # track of the release date and cleanup unrealistic sync entries after # some time. + + # TODO use cancellable deferreds instead locked = self.syncing_lock.locked() return locked diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 651d3ee5..9b546402 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -14,14 +14,10 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ A U1DB backend for encrypting data before sending to server and decrypting after receiving. """ - - import cStringIO import gzip import logging @@ -34,7 +30,7 @@ from time import sleep from uuid import uuid4 import simplejson as json -from taskthread import TimerTask + from u1db import errors from u1db.remote import utils, http_errors from u1db.remote.http_target import HTTPSyncTarget @@ -42,6 +38,8 @@ from u1db.remote.http_client import _encode_query_parameter, HTTPClientBase from zope.proxy import ProxyBase from zope.proxy import sameProxiedObjects, setProxiedObject +from twisted.internet.task import LoopingCall + from leap.soledad.common.document import SoledadDocument from leap.soledad.client.auth import TokenBasedAuth from leap.soledad.client.crypto import is_symmetrically_encrypted @@ -755,7 +753,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): """ Period of recurrence of the periodic decrypting task, in seconds. """ - DECRYPT_TASK_PERIOD = 0.5 + DECRYPT_LOOP_PERIOD = 0.5 # # Modified HTTPSyncTarget methods. @@ -802,7 +800,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): self._sync_db_write_lock = None self._decryption_callback = None self._sync_decr_pool = None - self._sync_watcher = None + self._sync_loop = None if sync_db and sync_db_write_lock is not None: self._sync_db = sync_db self._sync_db_write_lock = sync_db_write_lock @@ -828,23 +826,22 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): self._sync_decr_pool.close() self._sync_decr_pool = None - def _setup_sync_watcher(self): + def _setup_sync_loop(self): """ - Set up the sync watcher for deferred decryption. + Set up the sync loop for deferred decryption. """ - if self._sync_watcher is None: - self._sync_watcher = TimerTask( - self._decrypt_syncing_received_docs, - delay=self.DECRYPT_TASK_PERIOD) + if self._sync_loop is None: + self._sync_loop = LoopingCall( + self._decrypt_syncing_received_docs) + self._sync_loop.start(self.DECRYPT_LOOP_PERIOD) - def _teardown_sync_watcher(self): + def _teardown_sync_loop(self): """ - Tear down the sync watcher. + Tear down the sync loop. """ - if self._sync_watcher is not None: - self._sync_watcher.stop() - self._sync_watcher.shutdown() - self._sync_watcher = None + if self._sync_loop is not None: + self._sync_loop.stop() + self._sync_loop = None def _get_replica_uid(self, url): """ @@ -1131,7 +1128,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): if defer_decryption and self._sync_db is not None: self._sync_exchange_lock.acquire() self._setup_sync_decr_pool() - self._setup_sync_watcher() + self._setup_sync_loop() self._defer_decryption = True else: # fall back @@ -1292,10 +1289,10 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): # decrypt docs in case of deferred decryption if defer_decryption: - self._sync_watcher.start() + self._sync_loop.start() while self.clear_to_sync() is False: - sleep(self.DECRYPT_TASK_PERIOD) - self._teardown_sync_watcher() + sleep(self.DECRYPT_LOOP_PERIOD) + self._teardown_sync_loop() self._teardown_sync_decr_pool() self._sync_exchange_lock.release() @@ -1460,7 +1457,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): Decrypt the documents received from remote replica and insert them into the local one. - Called periodically from TimerTask self._sync_watcher. + Called periodically from LoopingCall self._sync_loop. """ if sameProxiedObjects( self._insert_doc_cb.get(self.source_replica_uid), diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py index 249cbdaa..970fac82 100644 --- a/common/src/leap/soledad/common/tests/util.py +++ b/common/src/leap/soledad/common/tests/util.py @@ -47,7 +47,8 @@ PASSWORD = '123456' def make_sqlcipher_database_for_test(test, replica_uid): - db = SQLCipherDatabase(':memory:', PASSWORD) + db = SQLCipherDatabase( + SQLCipherOptions(':memory:', PASSWORD)) db._set_replica_uid(replica_uid) return db -- cgit v1.2.3 From 1ae8f27c622034dc9524dab4b971bf0828966dd1 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 7 Oct 2014 11:32:23 -0300 Subject: Adapt sqlcipher tests to new api. --- client/src/leap/soledad/client/sqlcipher.py | 7 +- common/src/leap/soledad/common/tests/__init__.py | 2 +- .../leap/soledad/common/tests/test_sqlcipher.py | 551 +++------------------ .../soledad/common/tests/test_sqlcipher_sync.py | 409 +++++++++++++++ common/src/leap/soledad/common/tests/util.py | 7 +- 5 files changed, 493 insertions(+), 483 deletions(-) create mode 100644 common/src/leap/soledad/common/tests/test_sqlcipher_sync.py diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index a7e9e0fe..c645bb8d 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -502,9 +502,10 @@ class SQLCipherU1DBSync(object): else: sync_db_path = ":memory:" - # --------------------------------------------------------- - # TODO use a separate adbapi for this (sqlcipher only, no u1db) - # We could control that it only has 1 or 2 threads. + # XXX use initialize_sqlcipher_db here too + # TODO pass on_init queries to initialize_sqlcipher_db + self._sync_db = None#MPSafeSQLiteDB(sync_db_path) + pragmas.set_crypto_pragmas(self._sync_db, opts) opts.path = sync_db_path diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py index 0ab159fd..f8253409 100644 --- a/common/src/leap/soledad/common/tests/__init__.py +++ b/common/src/leap/soledad/common/tests/__init__.py @@ -92,7 +92,7 @@ class BaseSoledadTest(BaseLeapTest): def _soledad_instance(self, user=ADDRESS, passphrase=u'123', prefix='', - secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, + secrets_path='secrets.json', local_db_path='soledad.u1db', server_url='', cert_file=None, secret_id=None, shared_db_class=None): diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py index 273ac06e..78e2f01b 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py @@ -19,7 +19,6 @@ Test sqlcipher backend internals. """ import os import time -import simplejson as json import threading @@ -30,8 +29,6 @@ from pysqlcipher import dbapi2 from u1db import ( errors, query_parser, - sync, - vectorclock, ) from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase @@ -40,30 +37,31 @@ from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase from leap.soledad.common.document import SoledadDocument from leap.soledad.client.sqlcipher import ( SQLCipherDatabase, + SQLCipherOptions, DatabaseIsNotEncrypted, - open as u1db_open, + initialize_sqlcipher_db, ) -from leap.soledad.client.target import SoledadSyncTarget -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client.crypto import decrypt_doc_dict # u1db tests stuff. from leap.common.testing.basetest import BaseLeapTest -from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest +from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_sqlite_backend from leap.soledad.common.tests.u1db_tests import test_backends from leap.soledad.common.tests.u1db_tests import test_open -from leap.soledad.common.tests.u1db_tests import test_sync from leap.soledad.common.tests.util import ( make_sqlcipher_database_for_test, copy_sqlcipher_database_for_test, - make_soledad_app, - SoledadWithCouchServerMixin, PASSWORD, ) +def sqlcipher_open(path, passphrase, create=True, document_factory=None): + return SQLCipherDatabase( + None, + SQLCipherOptions(path, passphrase, create=create)) + + #----------------------------------------------------------------------------- # The following tests come from `u1db.tests.test_common_backend`. #----------------------------------------------------------------------------- @@ -71,7 +69,7 @@ from leap.soledad.common.tests.util import ( class TestSQLCipherBackendImpl(tests.TestCase): def test__allocate_doc_id(self): - db = SQLCipherDatabase(':memory:', PASSWORD) + db = sqlcipher_open(':memory:', PASSWORD) doc_id1 = db._allocate_doc_id() self.assertTrue(doc_id1.startswith('D-')) self.assertEqual(34, len(doc_id1)) @@ -129,6 +127,8 @@ class SQLCipherIndexTests(test_backends.DatabaseIndexTests): class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): def test_atomic_initialize(self): + # This test was modified to ensure that db2.close() is called within + # the thread that created the database. tmpdir = self.createTempDir() dbname = os.path.join(tmpdir, 'atomic.db') @@ -137,10 +137,12 @@ class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): class SQLCipherDatabaseTesting(SQLCipherDatabase): _index_storage_value = "testing" - def __init__(self, dbname, ntry): + def __init__(self, soledad_crypto, dbname, ntry): self._try = ntry self._is_initialized_invocations = 0 - SQLCipherDatabase.__init__(self, dbname, PASSWORD) + SQLCipherDatabase.__init__( + self, soledad_crypto, + SQLCipherOptions(dbname, PASSWORD)) def _is_initialized(self, c): res = \ @@ -153,25 +155,26 @@ class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): time.sleep(0.05) return res - outcome2 = [] + class SecondTry(threading.Thread): + + outcome2 = [] - def second_try(): - try: - db2 = SQLCipherDatabaseTesting(dbname, 2) - except Exception, e: - outcome2.append(e) - else: - outcome2.append(db2) + def run(self): + try: + db2 = SQLCipherDatabaseTesting(None, dbname, 2) + except Exception, e: + SecondTry.outcome2.append(e) + else: + SecondTry.outcome2.append(db2) + self.close() - t2 = threading.Thread(target=second_try) - db1 = SQLCipherDatabaseTesting(dbname, 1) + t2 = SecondTry() + db1 = SQLCipherDatabaseTesting(None, dbname, 1) t2.join() - self.assertIsInstance(outcome2[0], SQLCipherDatabaseTesting) - db2 = outcome2[0] - self.assertTrue(db2._is_initialized(db1._get_sqlite_handle().cursor())) + self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) + self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) db1.close() - db2.close() class TestAlternativeDocument(SoledadDocument): @@ -187,7 +190,7 @@ class TestSQLCipherPartialExpandDatabase( def setUp(self): test_sqlite_backend.TestSQLitePartialExpandDatabase.setUp(self) - self.db = SQLCipherDatabase(':memory:', PASSWORD) + self.db = sqlcipher_open(':memory:', PASSWORD) def tearDown(self): self.db.close() @@ -226,109 +229,64 @@ class TestSQLCipherPartialExpandDatabase( self.assertEqual('foo', self.db._replica_uid) def test__open_database(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/test.sqlite' - db1 = SQLCipherDatabase(path, PASSWORD) - db2 = SQLCipherDatabase._open_database(path, PASSWORD) - self.assertIsInstance(db2, SQLCipherDatabase) - db1.close() - db2.close() + # SQLCipherDatabase has no _open_database() method, so we just pass + # (and test for the same funcionality on test_open_database_existing() + # below). + pass def test__open_database_with_factory(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/test.sqlite' - db1 = SQLCipherDatabase(path, PASSWORD) - db2 = SQLCipherDatabase._open_database( - path, PASSWORD, - document_factory=TestAlternativeDocument) - doc = db2.create_doc({}) - self.assertTrue(isinstance(doc, SoledadDocument)) - db1.close() - db2.close() + # SQLCipherDatabase has no _open_database() method. + pass def test__open_database_non_existent(self): temp_dir = self.createTempDir(prefix='u1db-test-') path = temp_dir + '/non-existent.sqlite' self.assertRaises(errors.DatabaseDoesNotExist, - SQLCipherDatabase._open_database, - path, PASSWORD) + sqlcipher_open, + path, PASSWORD, create=False) def test__open_database_during_init(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/initialised.db' - db = SQLCipherDatabase.__new__( - SQLCipherDatabase) - db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed - db._sync_db = None - db._syncers = {} - db.sync_queue = None - c = db._db_handle.cursor() - c.execute('PRAGMA key="%s"' % PASSWORD) - self.addCleanup(db.close) - observed = [] - - class SQLiteDatabaseTesting(SQLCipherDatabase): - WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.1 - - @classmethod - def _which_index_storage(cls, c): - res = SQLCipherDatabase._which_index_storage(c) - db._ensure_schema() # init db - observed.append(res[0]) - return res - - db2 = SQLiteDatabaseTesting._open_database(path, PASSWORD) - self.addCleanup(db2.close) - self.assertIsInstance(db2, SQLCipherDatabase) - self.assertEqual( - [None, - SQLCipherDatabase._index_storage_value], - observed) - db.close() - db2.close() + # The purpose of this test is to ensure that _open_database() parallel + # db initialization behaviour is correct. As SQLCipherDatabase does + # not have an _open_database() method, we just do not implement this + # test. + pass def test__open_database_invalid(self): - class SQLiteDatabaseTesting(SQLCipherDatabase): - WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.1 + # This test was modified to ensure that an empty database file will + # raise a DatabaseIsNotEncrypted exception instead of a + # dbapi2.OperationalError exception. temp_dir = self.createTempDir(prefix='u1db-test-') path1 = temp_dir + '/invalid1.db' with open(path1, 'wb') as f: f.write("") - self.assertRaises(dbapi2.OperationalError, - SQLiteDatabaseTesting._open_database, path1, + self.assertRaises(DatabaseIsNotEncrypted, + sqlcipher_open, path1, PASSWORD) with open(path1, 'wb') as f: f.write("invalid") self.assertRaises(dbapi2.DatabaseError, - SQLiteDatabaseTesting._open_database, path1, + sqlcipher_open, path1, PASSWORD) def test_open_database_existing(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/existing.sqlite' - db1 = SQLCipherDatabase(path, PASSWORD) - db2 = SQLCipherDatabase.open_database(path, PASSWORD, create=False) - self.assertIsInstance(db2, SQLCipherDatabase) - db1.close() - db2.close() + # In the context of SQLCipherDatabase, where no _open_database() + # method exists and thus there's no call to _which_index_storage(), + # this test tests for the same functionality as + # test_open_database_create() below. So, we just pass. + pass def test_open_database_with_factory(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/existing.sqlite' - db1 = SQLCipherDatabase(path, PASSWORD) - db2 = SQLCipherDatabase.open_database( - path, PASSWORD, create=False, - document_factory=TestAlternativeDocument) - doc = db2.create_doc({}) - self.assertTrue(isinstance(doc, SoledadDocument)) - db1.close() - db2.close() + # SQLCipherDatabase's constructor has no factory parameter. + pass def test_open_database_create(self): + # SQLCipherDatabas has no open_database() method, so we just test for + # the actual database constructor effects. temp_dir = self.createTempDir(prefix='u1db-test-') path = temp_dir + '/new.sqlite' - db1 = SQLCipherDatabase.open_database(path, PASSWORD, create=True) - db2 = SQLCipherDatabase.open_database(path, PASSWORD, create=False) + db1 = sqlcipher_open(path, PASSWORD, create=True) + db2 = sqlcipher_open(path, PASSWORD, create=False) self.assertIsInstance(db2, SQLCipherDatabase) db1.close() db2.close() @@ -365,408 +323,47 @@ class TestSQLCipherPartialExpandDatabase( # The following tests come from `u1db.tests.test_open`. #----------------------------------------------------------------------------- + class SQLCipherOpen(test_open.TestU1DBOpen): def test_open_no_create(self): self.assertRaises(errors.DatabaseDoesNotExist, - u1db_open, self.db_path, - password=PASSWORD, + sqlcipher_open, self.db_path, + PASSWORD, create=False) self.assertFalse(os.path.exists(self.db_path)) def test_open_create(self): - db = u1db_open(self.db_path, password=PASSWORD, create=True) + db = sqlcipher_open(self.db_path, PASSWORD, create=True) self.addCleanup(db.close) self.assertTrue(os.path.exists(self.db_path)) self.assertIsInstance(db, SQLCipherDatabase) def test_open_with_factory(self): - db = u1db_open(self.db_path, password=PASSWORD, create=True, + db = sqlcipher_open(self.db_path, PASSWORD, create=True, document_factory=TestAlternativeDocument) self.addCleanup(db.close) doc = db.create_doc({}) self.assertTrue(isinstance(doc, SoledadDocument)) def test_open_existing(self): - db = SQLCipherDatabase(self.db_path, PASSWORD) + db = sqlcipher_open(self.db_path, PASSWORD) self.addCleanup(db.close) doc = db.create_doc_from_json(tests.simple_doc) # Even though create=True, we shouldn't wipe the db - db2 = u1db_open(self.db_path, password=PASSWORD, create=True) + db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) self.addCleanup(db2.close) doc2 = db2.get_doc(doc.doc_id) self.assertEqual(doc, doc2) def test_open_existing_no_create(self): - db = SQLCipherDatabase(self.db_path, PASSWORD) + db = sqlcipher_open(self.db_path, PASSWORD) self.addCleanup(db.close) - db2 = u1db_open(self.db_path, password=PASSWORD, create=False) + db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) self.addCleanup(db2.close) self.assertIsInstance(db2, SQLCipherDatabase) -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -#----------------------------------------------------------------------------- - -sync_scenarios = [] -for name, scenario in SQLCIPHER_SCENARIOS: - scenario = dict(scenario) - scenario['do_sync'] = test_sync.sync_via_synchronizer - sync_scenarios.append((name, scenario)) - scenario = dict(scenario) - - -def sync_via_synchronizer_and_leap(test, db_source, db_target, - trace_hook=None, trace_hook_shallow=None): - if trace_hook: - test.skipTest("full trace hook unsupported over http") - path = test._http_at[db_target] - target = SoledadSyncTarget.connect( - test.getURL(path), test._soledad._crypto) - target.set_token_credentials('user-uuid', 'auth-token') - if trace_hook_shallow: - target._set_trace_hook_shallow(trace_hook_shallow) - return sync.Synchronizer(db_source, target).sync() - - -sync_scenarios.append(('pyleap', { - 'make_database_for_test': test_sync.make_database_for_http_test, - 'copy_database_for_test': test_sync.copy_database_for_http_test, - 'make_document_for_test': make_document_for_test, - 'make_app_with_state': tests.test_remote_sync_target.make_http_app, - 'do_sync': test_sync.sync_via_synchronizer, -})) - - -class SQLCipherDatabaseSyncTests( - test_sync.DatabaseSyncTests, BaseSoledadTest): - """ - Test for succesfull sync between SQLCipher and LeapBackend. - - Some of the tests in this class had to be adapted because the remote - backend always receive encrypted content, and so it can not rely on - document's content comparison to try to autoresolve conflicts. - """ - - scenarios = sync_scenarios - - def setUp(self): - test_sync.DatabaseSyncTests.setUp(self) - - def tearDown(self): - test_sync.DatabaseSyncTests.tearDown(self) - if hasattr(self, 'db1') and isinstance(self.db1, SQLCipherDatabase): - self.db1.close() - if hasattr(self, 'db1_copy') \ - and isinstance(self.db1_copy, SQLCipherDatabase): - self.db1_copy.close() - if hasattr(self, 'db2') \ - and isinstance(self.db2, SQLCipherDatabase): - self.db2.close() - if hasattr(self, 'db2_copy') \ - and isinstance(self.db2_copy, SQLCipherDatabase): - self.db2_copy.close() - if hasattr(self, 'db3') \ - and isinstance(self.db3, SQLCipherDatabase): - self.db3.close() - - - - def test_sync_autoresolves(self): - """ - Test for sync autoresolve remote. - - This test was adapted because the remote database receives encrypted - content and so it can't compare documents contents to autoresolve. - """ - # The remote database can't autoresolve conflicts based on magic - # content convergence, so we modify this test to leave the possibility - # of the remode document ending up in conflicted state. - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev1 = doc1.rev - doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev2 = doc2.rev - self.sync(self.db1, self.db2) - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - # if remote content is in conflicted state, then document revisions - # will be different. - #self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) - v = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) - - def test_sync_autoresolves_moar(self): - """ - Test for sync autoresolve local. - - This test was adapted to decrypt remote content before assert. - """ - # here we test that when a database that has a conflicted document is - # the source of a sync, and the target database has a revision of the - # conflicted document that is newer than the source database's, and - # that target's database's document's content is the same as the - # source's document's conflict's, the source's document's conflict gets - # autoresolved, and the source's document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # `-------> - # a3b2 a1b2 (autoresolved) - # `-------> - # a3b2 a3b2 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - # because the conflict is on the source, sync it another time - self.sync(self.db1, self.db2) - # make sure db2 now has the exact same thing - doc1 = self.db1.get_doc('doc') - self.assertGetEncryptedDoc( - self.db2, - doc1.doc_id, doc1.rev, doc1.get_json(), False) - - def test_sync_autoresolves_moar_backwards(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # Despite that, in Soledad we suppose that the server never syncs, so - # it never has conflicted documents. Also, if it had, convergence - # would not be possible by checking document's contents because they - # would be encrypted in server. - # - # Therefore we suppress this test. - pass - - def test_sync_autoresolves_moar_backwards_three(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # We use the same reasoning from the last test to suppress this one. - pass - - def test_sync_propagates_resolution(self): - """ - Test if synchronization propagates resolution. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - db3 = self.create_database('test3', 'both') - self.sync(self.db2, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db2._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.sync(db3, self.db1) - # update on 2 - doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') - self.db2.put_doc(doc2) - self.sync(self.db2, db3) - self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) - # update on 1 - doc1.set_json('{"a": 3}') - self.db1.put_doc(doc1) - # conflicts - self.sync(self.db2, self.db1) - self.sync(db3, self.db1) - self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) - self.assertTrue(db3.get_doc('the-doc').has_conflicts) - # resolve - conflicts = self.db2.get_doc_conflicts('the-doc') - doc4 = self.make_document('the-doc', None, '{"a": 4}') - revs = [doc.rev for doc in conflicts] - self.db2.resolve_doc(doc4, revs) - doc2 = self.db2.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc2.get_json()) - self.assertFalse(doc2.has_conflicts) - self.sync(self.db2, db3) - doc3 = db3.get_doc('the-doc') - if ENC_SCHEME_KEY in doc3.content: - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc3.doc_id) - secret = _crypto.secret - doc3.set_json(decrypt_doc_dict( - doc3.content, - doc3.doc_id, doc3.rev, key, secret)) - self.assertEqual(doc4.get_json(), doc3.get_json()) - self.assertFalse(doc3.has_conflicts) - self.db1.close() - self.db2.close() - db3.close() - - def test_sync_puts_changes(self): - """ - Test if sync puts changes in remote replica. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(tests.simple_doc) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertGetEncryptedDoc( - self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - -def _make_local_db_and_token_http_target(test, path='test'): - test.startServer() - db = test.request_state._create_database(os.path.basename(path)) - st = SoledadSyncTarget.connect( - test.getURL(path), crypto=test._soledad._crypto) - st.set_token_credentials('user-uuid', 'auth-token') - return db, st - - -target_scenarios = [ - ('leap', { - 'create_db_and_target': _make_local_db_and_token_http_target, -# 'make_app_with_state': tests.test_remote_sync_target.make_http_app, - 'make_app_with_state': make_soledad_app, - 'do_sync': test_sync.sync_via_synchronizer}), -] - - -class SQLCipherSyncTargetTests( - SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): - - scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, - target_scenarios)) - - whitebox = False - - def setUp(self): - self.main_test_class = test_sync.DatabaseSyncTargetTests - SoledadWithCouchServerMixin.setUp(self) - - def test_sync_exchange(self): - """ - Modified to account for possibly receiving encrypted documents from - sever-side. - """ - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, - 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertTransactionLog(['doc-id'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, last_trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_push_many(self): - """ - Modified to account for possibly receiving encrypted documents from - sever-side. - """ - docs_by_gen = [ - (self.make_document( - 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), - (self.make_document('doc-id2', 'replica:1', tests.nested_doc), 11, - 'T-2')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) - self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(11, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_returns_many_new_docs(self): - """ - Modified to account for JSON serialization differences. - """ - doc = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db.create_doc_from_json(tests.nested_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - self.assertEqual(2, new_gen) - self.assertEqual( - [(doc.doc_id, doc.rev, 1), - (doc2.doc_id, doc2.rev, 2)], - [c[:2] + c[3:4] for c in self.other_changes]) - self.assertEqual( - json.dumps(tests.simple_doc), - json.dumps(self.other_changes[0][2])) - self.assertEqual( - json.loads(tests.nested_doc), - json.loads(self.other_changes[1][2])) - if self.whitebox: - self.assertEqual( - self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': - [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - - #----------------------------------------------------------------------------- # Tests for actual encryption of the database #----------------------------------------------------------------------------- @@ -792,7 +389,7 @@ class SQLCipherEncryptionTest(BaseLeapTest): """ SQLite backend should not succeed to open SQLCipher databases. """ - db = SQLCipherDatabase(self.DB_FILE, PASSWORD) + db = sqlcipher_open(self.DB_FILE, PASSWORD) doc = db.create_doc_from_json(tests.simple_doc) db.close() try: @@ -805,7 +402,7 @@ class SQLCipherEncryptionTest(BaseLeapTest): # at this point we know that the regular U1DB sqlcipher backend # did not succeed on opening the database, so it was indeed # encrypted. - db = SQLCipherDatabase(self.DB_FILE, PASSWORD) + db = sqlcipher_open(self.DB_FILE, PASSWORD) doc = db.get_doc(doc.doc_id) self.assertEqual(tests.simple_doc, doc.get_json(), 'decrypted content mismatch') @@ -822,7 +419,7 @@ class SQLCipherEncryptionTest(BaseLeapTest): try: # trying to open the a non-encrypted database with sqlcipher # backend should raise a DatabaseIsNotEncrypted exception. - db = SQLCipherDatabase(self.DB_FILE, PASSWORD) + db = sqlcipher_open(self.DB_FILE, PASSWORD) db.close() raise dbapi2.DatabaseError( "SQLCipher backend should not be able to open non-encrypted " diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py new file mode 100644 index 00000000..ad2a06b3 --- /dev/null +++ b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py @@ -0,0 +1,409 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test sqlcipher backend sync. +""" + + +import simplejson as json +from u1db import ( + sync, + vectorclock, +) + + +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.client.target import SoledadSyncTarget +from leap.soledad.client.crypto import decrypt_doc_dict +from leap.soledad.client.sqlcipher import ( + SQLCipherDatabase, +) + + +from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest +from leap.soledad.common.tests.u1db_tests import test_sync +from leap.soledad.common.tests.test_sqlcipher import ( + SQLCIPHER_SCENARIOS, + make_document_for_test, +) +from leap.soledad.common.tests.util import ( + make_soledad_app, + SoledadWithCouchServerMixin, +) + + +#----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +#----------------------------------------------------------------------------- + +sync_scenarios = [] +for name, scenario in SQLCIPHER_SCENARIOS: + scenario = dict(scenario) + scenario['do_sync'] = test_sync.sync_via_synchronizer + sync_scenarios.append((name, scenario)) + scenario = dict(scenario) + + +def sync_via_synchronizer_and_leap(test, db_source, db_target, + trace_hook=None, trace_hook_shallow=None): + if trace_hook: + test.skipTest("full trace hook unsupported over http") + path = test._http_at[db_target] + target = SoledadSyncTarget.connect( + test.getURL(path), test._soledad._crypto) + target.set_token_credentials('user-uuid', 'auth-token') + if trace_hook_shallow: + target._set_trace_hook_shallow(trace_hook_shallow) + return sync.Synchronizer(db_source, target).sync() + + +sync_scenarios.append(('pyleap', { + 'make_database_for_test': test_sync.make_database_for_http_test, + 'copy_database_for_test': test_sync.copy_database_for_http_test, + 'make_document_for_test': make_document_for_test, + 'make_app_with_state': tests.test_remote_sync_target.make_http_app, + 'do_sync': test_sync.sync_via_synchronizer, +})) + + +class SQLCipherDatabaseSyncTests( + test_sync.DatabaseSyncTests, BaseSoledadTest): + """ + Test for succesfull sync between SQLCipher and LeapBackend. + + Some of the tests in this class had to be adapted because the remote + backend always receive encrypted content, and so it can not rely on + document's content comparison to try to autoresolve conflicts. + """ + + scenarios = sync_scenarios + + def setUp(self): + test_sync.DatabaseSyncTests.setUp(self) + + def tearDown(self): + test_sync.DatabaseSyncTests.tearDown(self) + if hasattr(self, 'db1') and isinstance(self.db1, SQLCipherDatabase): + self.db1.close() + if hasattr(self, 'db1_copy') \ + and isinstance(self.db1_copy, SQLCipherDatabase): + self.db1_copy.close() + if hasattr(self, 'db2') \ + and isinstance(self.db2, SQLCipherDatabase): + self.db2.close() + if hasattr(self, 'db2_copy') \ + and isinstance(self.db2_copy, SQLCipherDatabase): + self.db2_copy.close() + if hasattr(self, 'db3') \ + and isinstance(self.db3, SQLCipherDatabase): + self.db3.close() + + + + def test_sync_autoresolves(self): + """ + Test for sync autoresolve remote. + + This test was adapted because the remote database receives encrypted + content and so it can't compare documents contents to autoresolve. + """ + # The remote database can't autoresolve conflicts based on magic + # content convergence, so we modify this test to leave the possibility + # of the remode document ending up in conflicted state. + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # if remote content is in conflicted state, then document revisions + # will be different. + #self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + """ + Test for sync autoresolve local. + + This test was adapted to decrypt remote content before assert. + """ + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + doc1 = self.db1.get_doc('doc') + self.assertGetEncryptedDoc( + self.db2, + doc1.doc_id, doc1.rev, doc1.get_json(), False) + + def test_sync_autoresolves_moar_backwards(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # Despite that, in Soledad we suppose that the server never syncs, so + # it never has conflicted documents. Also, if it had, convergence + # would not be possible by checking document's contents because they + # would be encrypted in server. + # + # Therefore we suppress this test. + pass + + def test_sync_autoresolves_moar_backwards_three(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # We use the same reasoning from the last test to suppress this one. + pass + + def test_sync_propagates_resolution(self): + """ + Test if synchronization propagates resolution. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, db3) + self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, db3) + doc3 = db3.get_doc('the-doc') + if ENC_SCHEME_KEY in doc3.content: + _crypto = self._soledad._crypto + key = _crypto.doc_passphrase(doc3.doc_id) + secret = _crypto.secret + doc3.set_json(decrypt_doc_dict( + doc3.content, + doc3.doc_id, doc3.rev, key, secret)) + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + self.db1.close() + self.db2.close() + db3.close() + + def test_sync_puts_changes(self): + """ + Test if sync puts changes in remote replica. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(tests.simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertGetEncryptedDoc( + self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + +def _make_local_db_and_token_http_target(test, path='test'): + test.startServer() + db = test.request_state._create_database(os.path.basename(path)) + st = SoledadSyncTarget.connect( + test.getURL(path), crypto=test._soledad._crypto) + st.set_token_credentials('user-uuid', 'auth-token') + return db, st + + +target_scenarios = [ + ('leap', { + 'create_db_and_target': _make_local_db_and_token_http_target, +# 'make_app_with_state': tests.test_remote_sync_target.make_http_app, + 'make_app_with_state': make_soledad_app, + 'do_sync': test_sync.sync_via_synchronizer}), +] + + +class SQLCipherSyncTargetTests( + SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): + + scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, + target_scenarios)) + + whitebox = False + + def setUp(self): + self.main_test_class = test_sync.DatabaseSyncTargetTests + SoledadWithCouchServerMixin.setUp(self) + + def test_sync_exchange(self): + """ + Modified to account for possibly receiving encrypted documents from + sever-side. + """ + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, + 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetEncryptedDoc( + self.db, 'doc-id', 'replica:1', tests.simple_doc, False) + self.assertTransactionLog(['doc-id'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, last_trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_push_many(self): + """ + Modified to account for possibly receiving encrypted documents from + sever-side. + """ + docs_by_gen = [ + (self.make_document( + 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), + (self.make_document('doc-id2', 'replica:1', tests.nested_doc), 11, + 'T-2')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetEncryptedDoc( + self.db, 'doc-id', 'replica:1', tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) + self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(11, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_returns_many_new_docs(self): + """ + Modified to account for JSON serialization differences. + """ + doc = self.db.create_doc_from_json(tests.simple_doc) + doc2 = self.db.create_doc_from_json(tests.nested_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + [(doc.doc_id, doc.rev, 1), + (doc2.doc_id, doc2.rev, 2)], + [c[:2] + c[3:4] for c in self.other_changes]) + self.assertEqual( + json.dumps(tests.simple_doc), + json.dumps(self.other_changes[0][2])) + self.assertEqual( + json.loads(tests.nested_doc), + json.loads(self.other_changes[1][2])) + if self.whitebox: + self.assertEqual( + self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': + [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) + diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py index 970fac82..b27e6356 100644 --- a/common/src/leap/soledad/common/tests/util.py +++ b/common/src/leap/soledad/common/tests/util.py @@ -36,11 +36,14 @@ from leap.soledad.server import SoledadApp from leap.soledad.server.auth import SoledadTokenAuthMiddleware -from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest +from leap.soledad.common.tests import BaseSoledadTest from leap.soledad.common.tests.test_couch import CouchDBWrapper, CouchDBTestCase -from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.sqlcipher import ( + SQLCipherDatabase, + SQLCipherOptions, +) PASSWORD = '123456' -- cgit v1.2.3 From 71d0ba384b16e5a1d9cfd4ee2b046ff6957f9b4e Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 9 Oct 2014 01:55:58 +0200 Subject: working sync-threadpool * Completed mapping of async dbpool * Fixed shared db initialization. Stuff To Be Fixed yet: [ ] All inserts have to be done from the sync threadpool. Right now we're reusing the connection from multiple threads in the syncer. I'm assuming the writes are automatically locking the file at the sqlite level, so this shouldn't pose a problem. [ ] Correctly handle the multiprocessing pool, and the callback execution. --- client/src/leap/soledad/client/adbapi.py | 44 +++------- client/src/leap/soledad/client/api.py | 72 ++++++++++------- client/src/leap/soledad/client/sqlcipher.py | 121 ++++++++++++++++++++++++++-- 3 files changed, 164 insertions(+), 73 deletions(-) diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index 60d9e195..733fce23 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -24,11 +24,9 @@ import sys from functools import partial -import u1db -from u1db.backends import sqlite_backend - from twisted.enterprise import adbapi from twisted.python import log +from zope.proxy import ProxyBase, setProxiedObject from leap.soledad.client import sqlcipher as soledad_sqlcipher @@ -46,39 +44,9 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): check_same_thread=False, cp_openfun=openfun) -class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase): - """ - A very simple wrapper for u1db around sqlcipher backend. - - Instead of initializing the database on the fly, it just uses an existing - connection that is passed to it in the initializer. - """ - - def __init__(self, conn): - self._db_handle = conn - self._real_replica_uid = None - self._ensure_schema() - self._factory = u1db.Document - - -class SoledadSQLCipherWrapper(soledad_sqlcipher.SQLCipherDatabase): - """ - A wrapper for u1db that uses the Soledad-extended sqlcipher backend. - - Instead of initializing the database on the fly, it just uses an existing - connection that is passed to it in the initializer. - """ - def __init__(self, conn): - self._db_handle = conn - self._real_replica_uid = None - self._ensure_schema() - self.set_document_factory(soledad_sqlcipher.soledad_doc_factory) - self._prime_replica_uid() - - class U1DBConnection(adbapi.Connection): - u1db_wrapper = SoledadSQLCipherWrapper + u1db_wrapper = soledad_sqlcipher.SoledadSQLCipherWrapper def __init__(self, pool, init_u1db=False): self.init_u1db = init_u1db @@ -120,6 +88,9 @@ class U1DBConnectionPool(adbapi.ConnectionPool): # all u1db connections, hashed by thread-id self.u1dbconnections = {} + # The replica uid, primed by the connections on init. + self.replica_uid = ProxyBase(None) + def runU1DBQuery(self, meth, *args, **kw): meth = "u1db_%s" % meth return self.runInteraction(self._runU1DBQuery, meth, *args, **kw) @@ -133,6 +104,11 @@ class U1DBConnectionPool(adbapi.ConnectionPool): u1db = self.u1dbconnections.get(tid) conn = self.connectionFactory(self, init_u1db=not bool(u1db)) + if self.replica_uid is None: + replica_uid = conn._u1db._real_replica_uid + setProxiedObject(self.replica_uid, replica_uid) + print "GOT REPLICA UID IN DBPOOL", self.replica_uid + if u1db is None: self.u1dbconnections[tid] = conn._u1db else: diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 493f6c1d..ff6257b2 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -158,10 +158,10 @@ class Soledad(object): # store config params self._uuid = uuid self._passphrase = passphrase - self._secrets_path = secrets_path self._local_db_path = local_db_path self._server_url = server_url self._defer_encryption = defer_encryption + self._secrets_path = None self.shared_db = None @@ -176,6 +176,8 @@ class Soledad(object): self._init_config_with_defaults() self._init_working_dirs() + self._secrets_path = secrets_path + # Initialize shared recovery database self.init_shared_db(server_url, uuid, self._creds) @@ -193,13 +195,12 @@ class Soledad(object): Initialize configuration using default values for missing params. """ soledad_assert_type(self._passphrase, unicode) - initialize = lambda attr, val: attr is None and setattr(attr, val) + initialize = lambda attr, val: getattr( + self, attr, None) is None and setattr(self, attr, val) - # initialize secrets_path - initialize(self._secrets_path, os.path.join( + initialize("_secrets_path", os.path.join( self.default_prefix, self.secrets_file_name)) - # initialize local_db_path - initialize(self._local_db_path, os.path.join( + initialize("_local_db_path", os.path.join( self.default_prefix, self.local_db_file_name)) # initialize server_url soledad_assert(self._server_url is not None, @@ -218,8 +219,8 @@ class Soledad(object): def _init_secrets(self): self._secrets = SoledadSecrets( - self.uuid, self.passphrase, self.secrets_path, - self._shared_db, self._crypto) + self.uuid, self._passphrase, self._secrets_path, + self.shared_db, self._crypto) self._secrets.bootstrap() def _init_u1db_sqlcipher_backend(self): @@ -249,8 +250,11 @@ class Soledad(object): self._dbpool = adbapi.getConnectionPool(opts) def _init_u1db_syncer(self): + replica_uid = self._dbpool.replica_uid + print "replica UID (syncer init)", replica_uid self._dbsyncer = SQLCipherU1DBSync( - self._soledad_opts, self._crypto, self._defer_encryption) + self._soledad_opts, self._crypto, replica_uid, + self._defer_encryption) # # Closing methods @@ -269,6 +273,9 @@ class Soledad(object): # ILocalStorage # + def _defer(self, meth, *args, **kw): + return self._dbpool.runU1DBQuery(meth, *args, **kw) + def put_doc(self, doc): """ ============================== WARNING ============================== @@ -282,58 +289,59 @@ class Soledad(object): # Isn't it better to defend ourselves from the mutability, to avoid # nasty surprises? doc.content = self._convert_to_unicode(doc.content) - return self._dbpool.put_doc(doc) + return self._defer("put_doc", doc) def delete_doc(self, doc): # XXX what does this do when fired??? - return self._dbpool.delete_doc(doc) + return self._defer("delete_doc", doc) def get_doc(self, doc_id, include_deleted=False): - return self._dbpool.get_doc(doc_id, include_deleted=include_deleted) + return self._defer( + "get_doc", doc_id, include_deleted=include_deleted) - def get_docs(self, doc_ids, check_for_conflicts=True, - include_deleted=False): - return self._dbpool.get_docs(doc_ids, - check_for_conflicts=check_for_conflicts, - include_deleted=include_deleted) + def get_docs( + self, doc_ids, check_for_conflicts=True, include_deleted=False): + return self._defer( + "get_docs", doc_ids, check_for_conflicts=check_for_conflicts, + include_deleted=include_deleted) def get_all_docs(self, include_deleted=False): - return self._dbpool.get_all_docs(include_deleted) + return self._defer("get_all_docs", include_deleted) def create_doc(self, content, doc_id=None): - return self._dbpool.create_doc( - _convert_to_unicode(content), doc_id=doc_id) + return self._defer( + "create_doc", _convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): - return self._dbpool.create_doc_from_json(json, doc_id=doc_id) + return self._defer("create_doc_from_json", json, doc_id=doc_id) def create_index(self, index_name, *index_expressions): - return self._dbpool.create_index(index_name, *index_expressions) + return self._defer("create_index", index_name, *index_expressions) def delete_index(self, index_name): - return self._dbpool.delete_index(index_name) + return self._defer("delete_index", index_name) def list_indexes(self): - return self._dbpool.list_indexes() + return self._defer("list_indexes") def get_from_index(self, index_name, *key_values): - return self._dbpool.get_from_index(index_name, *key_values) + return self._defer("get_from_index", index_name, *key_values) def get_count_from_index(self, index_name, *key_values): - return self._dbpool.get_count_from_index(index_name, *key_values) + return self._defer("get_count_from_index", index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): - return self._dbpool.get_range_from_index( - index_name, start_value, end_value) + return self._defer( + "get_range_from_index", index_name, start_value, end_value) def get_index_keys(self, index_name): - return self._dbpool.get_index_keys(index_name) + return self._defer("get_index_keys", index_name) def get_doc_conflicts(self, doc_id): - return self._dbpool.get_doc_conflicts(doc_id) + return self._defer("get_doc_conflicts", doc_id) def resolve_doc(self, doc, conflicted_doc_revs): - return self._dbpool.resolve_doc(doc, conflicted_doc_revs) + return self._defer("resolve_doc", doc, conflicted_doc_revs) def _get_local_db_path(self): return self._local_db_path @@ -460,6 +468,8 @@ class Soledad(object): # def init_shared_db(self, server_url, uuid, creds): + # XXX should assert that server_url begins with https + # Otherwise u1db target will fail. shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) self.shared_db = SoledadSharedDatabase.open_database( shared_db_url, diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index c645bb8d..c8e14176 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -55,10 +55,14 @@ from httplib import CannotSendRequest from pysqlcipher import dbapi2 as sqlcipher_dbapi2 from u1db.backends import sqlite_backend from u1db import errors as u1db_errors +import u1db + +from twisted.internet import reactor from twisted.internet.task import LoopingCall from twisted.internet.threads import deferToThreadPool from twisted.python.threadpool import ThreadPool +from twisted.python import log from leap.soledad.client import crypto from leap.soledad.client.target import SoledadSyncTarget @@ -77,7 +81,7 @@ logger = logging.getLogger(__name__) sqlite_backend.dbapi2 = sqlcipher_dbapi2 -def initialize_sqlcipher_db(opts, on_init=None): +def initialize_sqlcipher_db(opts, on_init=None, check_same_thread=True): """ Initialize a SQLCipher database. @@ -97,7 +101,7 @@ def initialize_sqlcipher_db(opts, on_init=None): raise u1db_errors.DatabaseDoesNotExist() conn = sqlcipher_dbapi2.connect( - opts.path) + opts.path, check_same_thread=check_same_thread) set_init_pragmas(conn, opts, extra_queries=on_init) return conn @@ -241,7 +245,6 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ self._real_replica_uid = None self._get_replica_uid() - print "REPLICA UID --->", self._real_replica_uid def _extra_schema_init(self, c): """ @@ -402,7 +405,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): self.close() -class SQLCipherU1DBSync(object): +class SQLCipherU1DBSync(SQLCipherDatabase): _sync_loop = None _sync_enc_pool = None @@ -435,7 +438,13 @@ class SQLCipherU1DBSync(object): def __init__(self, opts, soledad_crypto, replica_uid, defer_encryption=False): + self._opts = opts + self._path = opts.path self._crypto = soledad_crypto + self.__replica_uid = replica_uid + + print "REPLICA UID (u1dbsync init)", replica_uid + self._sync_db_key = opts.sync_db_key self._sync_db = None self._sync_db_write_lock = None @@ -453,9 +462,17 @@ class SQLCipherU1DBSync(object): self._sync_db_write_lock = threading.Lock() self.sync_queue = multiprocessing.Queue() + self.running = False self._sync_threadpool = None self._initialize_sync_threadpool() + self._reactor = reactor + self._reactor.callWhenRunning(self._start) + + self.ready = False + self._db_handle = None + self._initialize_syncer_main_db() + if defer_encryption: self._initialize_sync_db() @@ -476,6 +493,40 @@ class SQLCipherU1DBSync(object): self._sync_loop = LoopingCall(self._encrypt_syncing_docs), self._sync_loop.start(self.ENCRYPT_LOOP_PERIOD) + self.shutdownID = None + + @property + def _replica_uid(self): + return str(self.__replica_uid) + + def _start(self): + if not self.running: + self._sync_threadpool.start() + self.shutdownID = self._reactor.addSystemEventTrigger( + 'during', 'shutdown', self.finalClose) + self.running = True + + def _defer_to_sync_threadpool(self, meth, *args, **kwargs): + return deferToThreadPool( + self._reactor, self._sync_threadpool, meth, *args, **kwargs) + + def _initialize_syncer_main_db(self): + + def init_db(): + + # XXX DEBUG --------------------------------------------- + import thread + print "initializing in thread", thread.get_ident() + # XXX DEBUG --------------------------------------------- + + self._db_handle = initialize_sqlcipher_db( + self._opts, check_same_thread=False) + self._real_replica_uid = None + self._ensure_schema() + self.set_document_factory(soledad_doc_factory) + + return self._defer_to_sync_threadpool(init_db) + def _initialize_sync_threadpool(self): """ Initialize a ThreadPool with exactly one thread, that will be used to @@ -556,9 +607,19 @@ class SQLCipherU1DBSync(object): before the synchronisation was performed. :rtype: deferred """ + if not self.ready: + print "not ready yet..." + # XXX --------------------------------------------------------- + # This might happen because the database has not yet been + # initialized (it's deferred to the theadpool). + # A good strategy might involve to return a deferred that will + # callLater this same function after a timeout (deferLater) + # Might want to keep track of retries and cancel too. + # -------------------------------------------------------------- + print "Syncing to...", url kwargs = {'creds': creds, 'autocreate': autocreate, 'defer_decryption': defer_decryption} - return deferToThreadPool(self._sync, url, **kwargs) + return self._defer_to_sync_threadpool(self._sync, url, **kwargs) def _sync(self, url, creds=None, autocreate=True, defer_decryption=True): res = None @@ -568,9 +629,11 @@ class SQLCipherU1DBSync(object): # TODO review, I think this is no longer needed with a 1-thread # threadpool. + log.msg("in _sync") with self._syncer(url, creds=creds) as syncer: # XXX could mark the critical section here... try: + log.msg('syncer sync...') res = syncer.sync(autocreate=autocreate, defer_decryption=defer_decryption) @@ -590,6 +653,9 @@ class SQLCipherU1DBSync(object): """ Interrupt all ongoing syncs. """ + self._defer_to_sync_threadpool(self._stop_sync) + + def _stop_sync(self): for url in self._syncers: _, syncer = self._syncers[url] syncer.stop() @@ -604,13 +670,13 @@ class SQLCipherU1DBSync(object): Because of that, this method blocks until the syncing lock can be acquired. """ - with self.syncing_lock[self.replica_uid]: + with self.syncing_lock[self._path]: syncer = self._get_syncer(url, creds=creds) yield syncer @property def syncing(self): - lock = self.syncing_lock[self.replica_uid] + lock = self.syncing_lock[self._path] acquired_lock = lock.acquire(False) if acquired_lock is False: return True @@ -640,7 +706,8 @@ class SQLCipherU1DBSync(object): syncer = SoledadSynchronizer( self, SoledadSyncTarget(url, - self.replica_uid, + # XXX is the replica_uid ready? + self._replica_uid, creds=creds, crypto=self._crypto, sync_db=self._sync_db, @@ -689,6 +756,14 @@ class SQLCipherU1DBSync(object): # XXX this SHOULD BE a callback return self._get_generation() + def finalClose(self): + """ + This should only be called by the shutdown trigger. + """ + self.shutdownID = None + self._sync_threadpool.stop() + self.running = False + def close(self): """ Close the syncer and syncdb orderly @@ -718,6 +793,36 @@ class SQLCipherU1DBSync(object): self.sync_queue = None +class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase): + """ + A very simple wrapper for u1db around sqlcipher backend. + + Instead of initializing the database on the fly, it just uses an existing + connection that is passed to it in the initializer. + """ + + def __init__(self, conn): + self._db_handle = conn + self._real_replica_uid = None + self._ensure_schema() + self._factory = u1db.Document + + +class SoledadSQLCipherWrapper(SQLCipherDatabase): + """ + A wrapper for u1db that uses the Soledad-extended sqlcipher backend. + + Instead of initializing the database on the fly, it just uses an existing + connection that is passed to it in the initializer. + """ + def __init__(self, conn): + self._db_handle = conn + self._real_replica_uid = None + self._ensure_schema() + self.set_document_factory(soledad_doc_factory) + self._prime_replica_uid() + + def _assert_db_is_encrypted(opts): """ Assert that the sqlcipher file contains an encrypted database. -- cgit v1.2.3 From 133b72e2546ebabb1384583aec313e544aff69e2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 14 Oct 2014 18:42:08 +0200 Subject: add soledad sync example --- .../leap/soledad/client/examples/soledad_sync.py | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 client/src/leap/soledad/client/examples/soledad_sync.py diff --git a/client/src/leap/soledad/client/examples/soledad_sync.py b/client/src/leap/soledad/client/examples/soledad_sync.py new file mode 100644 index 00000000..6d0f6595 --- /dev/null +++ b/client/src/leap/soledad/client/examples/soledad_sync.py @@ -0,0 +1,65 @@ +from leap.bitmask.config.providerconfig import ProviderConfig +from leap.bitmask.crypto.srpauth import SRPAuth +from leap.soledad.client import Soledad + +import logging +logging.basicConfig(level=logging.DEBUG) + + +# EDIT THIS -------------------------------------------- +user = u"USERNAME" +uuid = u"USERUUID" +_pass = u"USERPASS" +server_url = "https://soledad.server.example.org:2323" +# EDIT THIS -------------------------------------------- + +secrets_path = "/tmp/%s.secrets" % uuid +local_db_path = "/tmp/%s.soledad" % uuid +cert_file = "/tmp/cacert.pem" +provider_config = '/tmp/cdev.json' + + +provider = ProviderConfig() +provider.load(provider_config) + +soledad = None + + +def printStuff(r): + print r + + +def printErr(err): + logging.exception(err.value) + + +def init_soledad(_): + token = srpauth.get_token() + print "token", token + + global soledad + soledad = Soledad(uuid, _pass, secrets_path, local_db_path, + server_url, cert_file, + auth_token=token, defer_encryption=False) + + def getall(_): + d = soledad.get_all_docs() + return d + + d1 = soledad.create_doc({"test": 42}) + d1.addCallback(getall) + d1.addCallbacks(printStuff, printErr) + + d2 = soledad.sync() + d2.addCallbacks(printStuff, printErr) + d2.addBoth(lambda r: reactor.stop()) + + +srpauth = SRPAuth(provider) + +d = srpauth.authenticate(user, _pass) +d.addCallbacks(init_soledad, printErr) + + +from twisted.internet import reactor +reactor.run() -- cgit v1.2.3 From d19e0d3b3b7a51d2e51800d41f53899254005661 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 16 Oct 2014 15:19:54 +0200 Subject: add syncable property to shared db --- client/src/leap/soledad/client/api.py | 20 ++++++---- client/src/leap/soledad/client/secrets.py | 59 +++++++++++++++++------------ client/src/leap/soledad/client/shared_db.py | 31 ++++++++++++--- 3 files changed, 73 insertions(+), 37 deletions(-) diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index ff6257b2..7886f397 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -113,7 +113,7 @@ class Soledad(object): def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, - auth_token=None, defer_encryption=False): + auth_token=None, defer_encryption=False, syncable=True): """ Initialize configuration, cryptographic keys and dbs. @@ -151,6 +151,11 @@ class Soledad(object): inline while syncing. :type defer_encryption: bool + :param syncable: + If set to ``False``, this database will not attempt to synchronize + with remote replicas (default is ``True``) + :type syncable: bool + :raise BootstrapSequenceError: Raised when the secret generation and storage on server sequence has failed for some reason. @@ -179,13 +184,15 @@ class Soledad(object): self._secrets_path = secrets_path # Initialize shared recovery database - self.init_shared_db(server_url, uuid, self._creds) + self.init_shared_db(server_url, uuid, self._creds, syncable=syncable) # The following can raise BootstrapSequenceError, that will be # propagated upwards. self._init_secrets() self._init_u1db_sqlcipher_backend() - self._init_u1db_syncer() + + if syncable: + self._init_u1db_syncer() # # initialization/destruction methods @@ -467,15 +474,14 @@ class Soledad(object): # ISharedSecretsStorage # - def init_shared_db(self, server_url, uuid, creds): - # XXX should assert that server_url begins with https - # Otherwise u1db target will fail. + def init_shared_db(self, server_url, uuid, creds, syncable=True): shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) self.shared_db = SoledadSharedDatabase.open_database( shared_db_url, uuid, creds=creds, - create=False) # db should exist at this point. + create=False, # db should exist at this point. + syncable=syncable) def _set_secrets_path(self, secrets_path): self._secrets.secrets_path = secrets_path diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 93f8c25d..81ccb114 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -289,9 +289,12 @@ class SoledadSecrets(object): :raises BootstrapSequenceError: Raised when unable to store secrets in shared database. """ - doc = self._get_secrets_from_shared_db() + if self._shared_db.syncable: + doc = self._get_secrets_from_shared_db() + else: + doc = None - if doc: + if doc is not None: logger.info( 'Found cryptographic secrets in shared recovery ' 'database.') @@ -308,21 +311,24 @@ class SoledadSecrets(object): 'No cryptographic secrets found, creating new ' ' secrets...') self.set_secret_id(self._gen_secret()) - try: - self._put_secrets_in_shared_db() - except Exception as ex: - # storing generated secret in shared db failed for - # some reason, so we erase the generated secret and - # raise. + + if self._shared_db.syncable: try: - os.unlink(self._secrets_path) - except OSError as e: - if e.errno != errno.ENOENT: # no such file or directory - logger.exception(e) - logger.exception(ex) - raise BootstrapSequenceError( - 'Could not store generated secret in the shared ' - 'database, bailing out...') + self._put_secrets_in_shared_db() + except Exception as ex: + # storing generated secret in shared db failed for + # some reason, so we erase the generated secret and + # raise. + try: + os.unlink(self._secrets_path) + except OSError as e: + if e.errno != errno.ENOENT: + # no such file or directory + logger.exception(e) + logger.exception(ex) + raise BootstrapSequenceError( + 'Could not store generated secret in the shared ' + 'database, bailing out...') # # Shared DB related methods @@ -434,7 +440,8 @@ class SoledadSecrets(object): 'contents.') # include secrets in the secret pool. secret_count = 0 - for secret_id, encrypted_secret in data[self.STORAGE_SECRETS_KEY].items(): + secrets = data[self.STORAGE_SECRETS_KEY].items() + for secret_id, encrypted_secret in secrets: if secret_id not in self._secrets: try: self._secrets[secret_id] = \ @@ -664,8 +671,8 @@ class SoledadSecrets(object): self._secrets_path = secrets_path secrets_path = property( - _get_secrets_path, - _set_secrets_path, + _get_secrets_path, + _set_secrets_path, doc='The path for the file containing the encrypted symmetric secret.') @property @@ -689,7 +696,7 @@ class SoledadSecrets(object): Return the secret for remote storage. """ key_start = 0 - key_end = self.REMOTE_STORAGE_SECRET_LENGTH + key_end = self.REMOTE_STORAGE_SECRET_LENGTH return self.storage_secret[key_start:key_end] # @@ -703,8 +710,10 @@ class SoledadSecrets(object): :return: The local storage secret. :rtype: str """ - pwd_start = self.REMOTE_STORAGE_SECRET_LENGTH + self.SALT_LENGTH - pwd_end = self.REMOTE_STORAGE_SECRET_LENGTH + self.LOCAL_STORAGE_SECRET_LENGTH + secret_len = self.REMOTE_STORAGE_SECRET_LENGTH + lsecret_len = self.LOCAL_STORAGE_SECRET_LENGTH + pwd_start = secret_len + self.SALT_LENGTH + pwd_end = secret_len + lsecret_len return self.storage_secret[pwd_start:pwd_end] def _get_local_storage_salt(self): @@ -731,9 +740,9 @@ class SoledadSecrets(object): buflen=32, # we need a key with 256 bits (32 bytes) ) - # - # sync db key - # + # + # sync db key + # def _get_sync_db_salt(self): """ diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index 31c4e8e8..7ec71991 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -26,6 +26,9 @@ from leap.soledad.client.auth import TokenBasedAuth # Soledad shared database # ---------------------------------------------------------------------------- +# TODO could have a hierarchy of soledad exceptions. + + class NoTokenForAuth(Exception): """ No token was found for token-based authentication. @@ -38,6 +41,12 @@ class Unauthorized(Exception): """ +class ImproperlyConfiguredError(Exception): + """ + Wrong parameters in the database configuration. + """ + + class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): """ This is a shared recovery database that enables users to store their @@ -46,6 +55,8 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): # TODO: prevent client from messing with the shared DB. # TODO: define and document API. + syncable = True + # # Token auth methods. # @@ -82,7 +93,7 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): # @staticmethod - def open_database(url, uuid, create, creds=None): + def open_database(url, uuid, create, creds=None, syncable=True): # TODO: users should not be able to create the shared database, so we # have to remove this from here in the future. """ @@ -101,8 +112,13 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :return: The shared database in the given url. :rtype: SoledadSharedDatabase """ + if syncable and not url.startswith('https://'): + raise ImproperlyConfiguredError( + "Remote soledad server must be an https URI") db = SoledadSharedDatabase(url, uuid, creds=creds) - db.open(create) + db.syncable = syncable + if syncable: + db.open(create) return db @staticmethod @@ -145,9 +161,14 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :raise HTTPError: Raised if any HTTP error occurs. """ - res, headers = self._request_json('PUT', ['lock', self._uuid], - body={}) - return res['token'], res['timeout'] + # TODO ----- if the shared_db is not syncable, should not + # attempt to resolve. + if self.syncable: + res, headers = self._request_json( + 'PUT', ['lock', self._uuid], body={}) + return res['token'], res['timeout'] + else: + return None, None def unlock(self, token): """ -- cgit v1.2.3 From 092f8b784a260121254a20235fa7aa41b05212e1 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 21 Oct 2014 10:16:43 +0200 Subject: minor naming/documentation fixes after drebs review --- client/src/leap/soledad/client/adbapi.py | 8 ++-- client/src/leap/soledad/client/api.py | 52 ++-------------------- .../src/leap/soledad/client/examples/use_adbapi.py | 4 +- client/src/leap/soledad/client/examples/use_api.py | 4 +- client/src/leap/soledad/client/interfaces.py | 15 ++++--- client/src/leap/soledad/client/shared_db.py | 9 +++- client/src/leap/soledad/client/sqlcipher.py | 35 ++++++++------- 7 files changed, 47 insertions(+), 80 deletions(-) diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index 733fce23..0cdc90eb 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -86,7 +86,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): def __init__(self, *args, **kwargs): adbapi.ConnectionPool.__init__(self, *args, **kwargs) # all u1db connections, hashed by thread-id - self.u1dbconnections = {} + self._u1dbconnections = {} # The replica uid, primed by the connections on init. self.replica_uid = ProxyBase(None) @@ -101,7 +101,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): def _runInteraction(self, interaction, *args, **kw): tid = self.threadID() - u1db = self.u1dbconnections.get(tid) + u1db = self._u1dbconnections.get(tid) conn = self.connectionFactory(self, init_u1db=not bool(u1db)) if self.replica_uid is None: @@ -110,7 +110,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): print "GOT REPLICA UID IN DBPOOL", self.replica_uid if u1db is None: - self.u1dbconnections[tid] = conn._u1db + self._u1dbconnections[tid] = conn._u1db else: conn._u1db = u1db @@ -134,6 +134,6 @@ class U1DBConnectionPool(adbapi.ConnectionPool): self.running = False for conn in self.connections.values(): self._close(conn) - for u1db in self.u1dbconnections.values(): + for u1db in self._u1dbconnections.values(): self._close(u1db) self.connections.clear() diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 7886f397..00884a12 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -105,7 +105,7 @@ class Soledad(object): """ implements(soledad_interfaces.ILocalStorage, soledad_interfaces.ISyncableStorage, - soledad_interfaces.ISharedSecretsStorage) + soledad_interfaces.ISecretsStorage) local_db_file_name = 'soledad.u1db' secrets_file_name = "soledad.json" @@ -292,10 +292,7 @@ class Soledad(object): call. ============================== WARNING ============================== """ - # TODO what happens with this warning during the deferred life cycle? - # Isn't it better to defend ourselves from the mutability, to avoid - # nasty surprises? - doc.content = self._convert_to_unicode(doc.content) + doc.content = _convert_to_unicode(doc.content) return self._defer("put_doc", doc) def delete_doc(self, doc): @@ -388,7 +385,7 @@ class Soledad(object): soledad_events.SOLEDAD_DONE_DATA_SYNC, self.uuid) return local_gen - sync_url = urlparse.urljoin(self.server_url, 'user-%s' % self.uuid) + sync_url = urlparse.urljoin(self._server_url, 'user-%s' % self.uuid) try: d = self._dbsyncer.sync( sync_url, @@ -405,29 +402,6 @@ class Soledad(object): def stop_sync(self): self._dbsyncer.stop_sync() - # FIXME ------------------------------------------------------- - # review if we really need this. I think that we can the sync - # fail silently if nothing is to be synced. - #def need_sync(self, url): - # XXX dispatch this method in the dbpool ................. - #replica_uid = self._dbpool.replica_uid - #target = SoledadSyncTarget( - #url, replica_uid, creds=self._creds, crypto=self._crypto) -# - # XXX does it matter if we get this from the general dbpool or the - # syncer pool? - #generation = self._dbpool.get_generation() -# - # XXX better unpack it? - #info = target.get_sync_info(replica_uid) -# - # compare source generation with target's last known source generation - #if generation != info[4]: - #soledad_events.signal( - #soledad_events.SOLEDAD_NEW_DATA_TO_SYNC, self.uuid) - #return True - #return False - @property def syncing(self): return self._dbsyncer.syncing @@ -463,15 +437,8 @@ class Soledad(object): token = property(_get_token, _set_token, doc='The authentication Token.') - def _get_server_url(self): - return self._server_url - - server_url = property( - _get_server_url, - doc='The URL of the Soledad server.') - # - # ISharedSecretsStorage + # ISecretsStorage # def init_shared_db(self, server_url, uuid, creds, syncable=True): @@ -483,17 +450,6 @@ class Soledad(object): create=False, # db should exist at this point. syncable=syncable) - def _set_secrets_path(self, secrets_path): - self._secrets.secrets_path = secrets_path - - def _get_secrets_path(self): - return self._secrets.secrets_path - - secrets_path = property( - _get_secrets_path, - _set_secrets_path, - doc='The path for the file containing the encrypted symmetric secret.') - @property def storage_secret(self): """ diff --git a/client/src/leap/soledad/client/examples/use_adbapi.py b/client/src/leap/soledad/client/examples/use_adbapi.py index d3ee8527..d7bd21f2 100644 --- a/client/src/leap/soledad/client/examples/use_adbapi.py +++ b/client/src/leap/soledad/client/examples/use_adbapi.py @@ -88,10 +88,10 @@ def allDone(_): reactor.stop() deferreds = [] +payload = open('manifest.phk').read() for i in range(times): - doc = {"number": i, - "payload": open('manifest.phk').read()} + doc = {"number": i, "payload": payload} d = createDoc(doc) d.addCallbacks(printResult, lambda e: e.printTraceback()) deferreds.append(d) diff --git a/client/src/leap/soledad/client/examples/use_api.py b/client/src/leap/soledad/client/examples/use_api.py index 4268fe71..e2501c98 100644 --- a/client/src/leap/soledad/client/examples/use_api.py +++ b/client/src/leap/soledad/client/examples/use_api.py @@ -52,10 +52,10 @@ db = sqlcipher.SQLCipherDatabase(opts) def allDone(): debug("ALL DONE!") +payload = open('manifest.phk').read() for i in range(times): - doc = {"number": i, - "payload": open('manifest.phk').read()} + doc = {"number": i, "payload": payload} d = db.create_doc(doc) debug(d.doc_id, d.content['number']) diff --git a/client/src/leap/soledad/client/interfaces.py b/client/src/leap/soledad/client/interfaces.py index 6bd3f200..4f7b0779 100644 --- a/client/src/leap/soledad/client/interfaces.py +++ b/client/src/leap/soledad/client/interfaces.py @@ -22,7 +22,8 @@ from zope.interface import Interface, Attribute class ILocalStorage(Interface): """ - I implement core methods for the u1db local storage. + I implement core methods for the u1db local storage of documents and + indexes. """ local_db_path = Attribute( "The path for the local database replica") @@ -285,7 +286,6 @@ class ISyncableStorage(Interface): I implement methods to synchronize with a remote replica. """ replica_uid = Attribute("The uid of the local replica") - server_url = Attribute("The URL of the Soledad server.") syncing = Attribute( "Property, True if the syncer is syncing.") token = Attribute("The authentication Token.") @@ -317,12 +317,11 @@ class ISyncableStorage(Interface): """ -class ISharedSecretsStorage(Interface): +class ISecretsStorage(Interface): """ - I implement methods needed for the Shared Recovery Database. + I implement methods needed for initializing and accessing secrets, that are + synced against the Shared Recovery Database. """ - secrets_path = Attribute( - "Path for storing encrypted key used for symmetric encryption.") secrets_file_name = Attribute( "The name of the file where the storage secrets will be stored") @@ -332,7 +331,9 @@ class ISharedSecretsStorage(Interface): # XXX this used internally from secrets, so it might be good to preserve # as a public boundary with other components. - secrets = Attribute("") + + # We should also probably document its interface. + secrets = Attribute("A SoledadSecrets object containing access to secrets") def init_shared_db(self, server_url, uuid, creds): """ diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index 7ec71991..77a7db68 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -55,6 +55,8 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): # TODO: prevent client from messing with the shared DB. # TODO: define and document API. + # If syncable is False, the database will not attempt to sync against + # a remote replica. Default is True. syncable = True # @@ -109,6 +111,11 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :param token: An authentication token for accessing the shared db. :type token: str + :param syncable: + If syncable is False, the database will not attempt to sync against + a remote replica. + :type syncable: bool + :return: The shared database in the given url. :rtype: SoledadSharedDatabase """ @@ -161,8 +168,6 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :raise HTTPError: Raised if any HTTP error occurs. """ - # TODO ----- if the shared_db is not syncable, should not - # attempt to resolve. if self.syncable: res, headers = self._request_json( 'PUT', ['lock', self._uuid], body={}) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index c8e14176..323d78f1 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -69,7 +69,6 @@ from leap.soledad.client.target import SoledadSyncTarget from leap.soledad.client.target import PendingReceivedDocsSyncError from leap.soledad.client.sync import SoledadSynchronizer -# TODO use adbapi too from leap.soledad.client import pragmas from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument @@ -115,7 +114,7 @@ def set_init_pragmas(conn, opts=None, extra_queries=None): This includes the crypto pragmas, and any other options that must be passed early to sqlcipher db. """ - assert opts is not None + soledad_assert(opts is not None) extra_queries = [] if extra_queries is None else extra_queries with _db_init_lock: # only one execution path should initialize the db @@ -196,8 +195,8 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ defer_encryption = False - # The attribute _index_storage_value will be used as the lookup key. - # Here we extend it with `encrypted` + # The attribute _index_storage_value will be used as the lookup key for the + # implementation of the SQLCipher storage backend. _index_storage_value = 'expand referenced encrypted' def __init__(self, opts): @@ -227,7 +226,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): # TODO --------------------------------------------------- # Everything else in this initialization has to be factored - # out, so it can be used from U1DBSqlcipherWrapper __init__ + # out, so it can be used from SoledadSQLCipherWrapper.__init__ # too. # --------------------------------------------------------- @@ -406,6 +405,9 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): class SQLCipherU1DBSync(SQLCipherDatabase): + """ + Soledad syncer implementation. + """ _sync_loop = None _sync_enc_pool = None @@ -454,7 +456,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase): # we store syncers in a dictionary indexed by the target URL. We also # store a hash of the auth info in case auth info expires and we need # to rebuild the syncer for that target. The final self._syncers - # format is the following:: + # format is the following: # # self._syncers = {'': ('', syncer), ...} @@ -514,10 +516,12 @@ class SQLCipherU1DBSync(SQLCipherDatabase): def init_db(): - # XXX DEBUG --------------------------------------------- - import thread - print "initializing in thread", thread.get_ident() - # XXX DEBUG --------------------------------------------- + # XXX DEBUG ----------------------------------------- + # REMOVE ME when merging. + + #import thread + #print "initializing in thread", thread.get_ident() + # --------------------------------------------------- self._db_handle = initialize_sqlcipher_db( self._opts, check_same_thread=False) @@ -553,11 +557,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): else: sync_db_path = ":memory:" - # XXX use initialize_sqlcipher_db here too - # TODO pass on_init queries to initialize_sqlcipher_db - self._sync_db = None#MPSafeSQLiteDB(sync_db_path) - pragmas.set_crypto_pragmas(self._sync_db, opts) - opts.path = sync_db_path self._sync_db = initialize_sqlcipher_db( @@ -799,6 +798,9 @@ class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase): Instead of initializing the database on the fly, it just uses an existing connection that is passed to it in the initializer. + + It can be used in tests and debug runs to initialize the adbapi with plain + sqlite connections, decoupled from the sqlcipher layer. """ def __init__(self, conn): @@ -814,6 +816,9 @@ class SoledadSQLCipherWrapper(SQLCipherDatabase): Instead of initializing the database on the fly, it just uses an existing connection that is passed to it in the initializer. + + It can be used from adbapi to initialize a soledad database after + getting a regular connection to a sqlcipher database. """ def __init__(self, conn): self._db_handle = conn -- cgit v1.2.3 From d25527ac06563f061aee7771d494522b3ed58b7d Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 18 Nov 2014 14:14:42 -0200 Subject: Save active secret on recovery document. --- client/src/leap/soledad/client/secrets.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 81ccb114..b0e54220 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -132,6 +132,7 @@ class SoledadSecrets(object): UUID_KEY = 'uuid' STORAGE_SECRETS_KEY = 'storage_secrets' + ACTIVE_SECRET_KEY = 'active_secret' SECRET_KEY = 'secret' CIPHER_KEY = 'cipher' LENGTH_KEY = 'length' @@ -265,10 +266,13 @@ class SoledadSecrets(object): content = None with open(self._secrets_path, 'r') as f: content = json.loads(f.read()) - _, mac = self._import_recovery_document(content) + _, mac, active_secret = self._import_recovery_document(content) # choose first secret if no secret_id was given if self._secret_id is None: - self.set_secret_id(self._secrets.items()[0][0]) + if active_secret is None: + self.set_secret_id(self._secrets.items()[0][0]) + else: + self.set_secret_id(active_secret) # enlarge secret if needed enlarged = False if len(self._secrets[self._secret_id]) < self.GEN_SECRET_LENGTH: @@ -298,12 +302,15 @@ class SoledadSecrets(object): logger.info( 'Found cryptographic secrets in shared recovery ' 'database.') - _, mac = self._import_recovery_document(doc.content) + _, mac, active_secret = self._import_recovery_document(doc.content) if mac is False: self.put_secrets_in_shared_db() self._store_secrets() # save new secrets in local file if self._secret_id is None: - self.set_secret_id(self._secrets.items()[0][0]) + if active_secret is None: + self.set_secret_id(self._secrets.items()[0][0]) + else: + self.set_secret_id(active_secret) else: # STAGE 3 - there are no secrets in server also, so # generate a secret and store it in remote db. @@ -363,6 +370,7 @@ class SoledadSecrets(object): 'secret': '', }, }, + 'active_secret': '', 'kdf': 'scrypt', 'kdf_salt': '', 'kdf_length: , @@ -388,6 +396,7 @@ class SoledadSecrets(object): # create the recovery document data = { self.STORAGE_SECRETS_KEY: encrypted_secrets, + self.ACTIVE_SECRET_KEY: self._secret_id, self.KDF_KEY: self.KDF_SCRYPT, self.KDF_SALT_KEY: binascii.b2a_base64(salt), self.KDF_LENGTH_KEY: len(key), @@ -410,8 +419,9 @@ class SoledadSecrets(object): :param data: The recovery document. :type data: dict - :return: A tuple containing the number of imported secrets and whether - there was MAC informationa available for authenticating. + :return: A tuple containing the number of imported secrets, whether + there was MAC information available for authenticating, and + the secret_id of the last active secret. :rtype: (int, bool) """ soledad_assert(self.STORAGE_SECRETS_KEY in data) @@ -441,6 +451,11 @@ class SoledadSecrets(object): # include secrets in the secret pool. secret_count = 0 secrets = data[self.STORAGE_SECRETS_KEY].items() + active_secret = None + # XXX remove check for existence of key (included for backwards + # compatibility) + if self.ACTIVE_SECRET_KEY in data: + active_secret = data[self.ACTIVE_SECRET_KEY] for secret_id, encrypted_secret in secrets: if secret_id not in self._secrets: try: @@ -450,7 +465,7 @@ class SoledadSecrets(object): except SecretsException as e: logger.error("Failed to decrypt storage secret: %s" % str(e)) - return secret_count, mac + return secret_count, mac, active_secret def _get_secrets_from_shared_db(self): """ -- cgit v1.2.3 From 8b3982ada921af765e7ede7dd3c77ef3fbf075f1 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 18 Nov 2014 14:21:58 -0200 Subject: Standardize export of secrets to avoid miscalculation of MAC. --- client/src/leap/soledad/client/secrets.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index b0e54220..af781a26 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -403,7 +403,7 @@ class SoledadSecrets(object): crypto.MAC_METHOD_KEY: crypto.MacMethods.HMAC, crypto.MAC_KEY: hmac.new( key, - json.dumps(encrypted_secrets), + json.dumps(encrypted_secrets, sort_keys=True), sha256).hexdigest(), } return data @@ -440,7 +440,8 @@ class SoledadSecrets(object): buflen=32) mac = hmac.new( key, - json.dumps(data[self.STORAGE_SECRETS_KEY]), + json.dumps( + data[self.STORAGE_SECRETS_KEY], sort_keys=True), sha256).hexdigest() else: raise crypto.UnknownMacMethodError('Unknown MAC method: %s.' % -- cgit v1.2.3 From 4b317c9f5a9033afaa7435e11f761de4bc3095a3 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 18 Nov 2014 15:33:23 -0200 Subject: Fix interruptable sync. --- client/src/leap/soledad/client/target.py | 34 ++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index 9b546402..ba61cdff 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -348,7 +348,7 @@ class DocumentSyncerPool(object): self._threads.remove(syncer_thread) self._semaphore_pool.release() - def cancel_threads(self, calling_thread): + def cancel_threads(self): """ Stop all threads in the pool. """ @@ -794,6 +794,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): self._sync_exchange_lock = threading.Lock() self.source_replica_uid = source_replica_uid self._defer_decryption = False + self._syncer_pool = None # deferred decryption attributes self._sync_db = None @@ -952,7 +953,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): def _get_remote_docs(self, url, last_known_generation, last_known_trans_id, headers, return_doc_cb, ensure_callback, sync_id, - syncer_pool, defer_decryption=False): + defer_decryption=False): """ Fetch sync documents from the remote database and insert them in the local database. @@ -1013,7 +1014,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): break # launch a thread to fetch one document from target - t = syncer_pool.new_syncer_thread( + t = self._syncer_pool.new_syncer_thread( idx, number_of_changes, last_callback_lock=last_callback_lock) @@ -1047,6 +1048,8 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): t.join() if t.success: number_of_changes, _, _ = t.result + else: + raise t.exception first_request = False # make sure all threads finished and we have up-to-date info @@ -1057,6 +1060,8 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): t.join() if t.success: last_successful_thread = t + else: + raise t.exception # get information about last successful thread if last_successful_thread is not None: @@ -1162,9 +1167,9 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): logger.debug("Soledad sync send status: %s" % msg) defer_encryption = self._sync_db is not None - syncer_pool = DocumentSyncerPool( + self._syncer_pool = DocumentSyncerPool( self._raw_url, self._raw_creds, url, headers, ensure_callback, - self.stop) + self.stop_syncer) threads = [] last_callback_lock = None sent = 0 @@ -1209,7 +1214,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): # ------------------------------------------------------------- # end of symmetric encryption # ------------------------------------------------------------- - t = syncer_pool.new_syncer_thread( + t = self._syncer_pool.new_syncer_thread( sent + 1, total, last_request_lock=last_request_lock, last_callback_lock=last_callback_lock) @@ -1264,6 +1269,8 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): if t.success: synced.append((doc.doc_id, doc.rev)) last_successful_thread = t + else: + raise t.exception # delete documents from the sync database if defer_encryption: @@ -1282,10 +1289,10 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): cur_target_gen, cur_target_trans_id = self._get_remote_docs( url, last_known_generation, last_known_trans_id, headers, - return_doc_cb, ensure_callback, sync_id, syncer_pool, + return_doc_cb, ensure_callback, sync_id, defer_decryption=defer_decryption) - syncer_pool.cleanup() + self._syncer_pool.cleanup() # decrypt docs in case of deferred decryption if defer_decryption: @@ -1303,6 +1310,7 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): cur_target_trans_id = trans_id_after_send self.stop() + self._syncer_pool = None return cur_target_gen, cur_target_trans_id def start(self): @@ -1312,6 +1320,11 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): with self._stop_lock: self._stopped = False + + def stop_syncer(self): + with self._stop_lock: + self._stopped = True + def stop(self): """ Mark current sync session as stopped. @@ -1320,8 +1333,9 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): enough information to the synchronizer so the sync session can be recovered afterwards. """ - with self._stop_lock: - self._stopped = True + self.stop_syncer() + if self._syncer_pool: + self._syncer_pool.cancel_threads() @property def stopped(self): -- cgit v1.2.3 From b915e3d5bd1e37c732b44559af5587f6c6a90fc3 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 14:58:31 -0200 Subject: Adapt tests for new api. --- common/src/leap/soledad/common/tests/__init__.py | 284 +------- common/src/leap/soledad/common/tests/test_couch.py | 164 +---- .../tests/test_couch_operations_atomicity.py | 206 +++--- .../src/leap/soledad/common/tests/test_crypto.py | 17 +- common/src/leap/soledad/common/tests/test_http.py | 5 - .../leap/soledad/common/tests/test_http_client.py | 11 +- common/src/leap/soledad/common/tests/test_https.py | 42 +- .../src/leap/soledad/common/tests/test_server.py | 352 ++++----- .../src/leap/soledad/common/tests/test_soledad.py | 166 ++--- .../leap/soledad/common/tests/test_soledad_app.py | 59 ++ .../leap/soledad/common/tests/test_soledad_doc.py | 24 +- .../leap/soledad/common/tests/test_sqlcipher.py | 62 +- .../soledad/common/tests/test_sqlcipher_sync.py | 49 +- common/src/leap/soledad/common/tests/test_sync.py | 153 ++-- .../soledad/common/tests/test_sync_deferred.py | 150 ++-- .../leap/soledad/common/tests/test_sync_target.py | 285 ++++---- .../src/leap/soledad/common/tests/test_target.py | 797 --------------------- .../soledad/common/tests/test_target_soledad.py | 102 --- .../soledad/common/tests/u1db_tests/__init__.py | 4 +- .../soledad/common/tests/u1db_tests/test_https.py | 2 +- common/src/leap/soledad/common/tests/util.py | 352 ++++++++- 21 files changed, 1116 insertions(+), 2170 deletions(-) create mode 100644 common/src/leap/soledad/common/tests/test_soledad_app.py delete mode 100644 common/src/leap/soledad/common/tests/test_target.py delete mode 100644 common/src/leap/soledad/common/tests/test_target_soledad.py diff --git a/common/src/leap/soledad/common/tests/__init__.py b/common/src/leap/soledad/common/tests/__init__.py index f8253409..acebb77b 100644 --- a/common/src/leap/soledad/common/tests/__init__.py +++ b/common/src/leap/soledad/common/tests/__init__.py @@ -19,291 +19,9 @@ """ Tests to make sure Soledad provides U1DB functionality and more. """ -import os -import random -import string -import u1db -from mock import Mock - - -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client import Soledad -from leap.soledad.client.crypto import decrypt_doc_dict -from leap.common.testing.basetest import BaseLeapTest - - -#----------------------------------------------------------------------------- -# Some tests inherit from BaseSoledadTest in order to have a working Soledad -# instance in each test. -#----------------------------------------------------------------------------- - -ADDRESS = 'leap@leap.se' - - -class BaseSoledadTest(BaseLeapTest): - """ - Instantiates Soledad for usage in tests. - """ - defer_sync_encryption = False - - def setUp(self): - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - self.db2_file = os.path.join(self.tempdir, "db2.u1db") - self.email = ADDRESS - # open test dbs - self._db1 = u1db.open(self.db1_file, create=True, - document_factory=SoledadDocument) - self._db2 = u1db.open(self.db2_file, create=True, - document_factory=SoledadDocument) - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - # initialize soledad by hand so we can control keys - self._soledad = self._soledad_instance( - prefix=self.rand_prefix, user=self.email) - - def tearDown(self): - self._db1.close() - self._db2.close() - self._soledad.close() - - # XXX should not access "private" attrs - for f in [self._soledad._local_db_path, self._soledad._secrets_path]: - if os.path.isfile(f): - os.unlink(f) - def get_default_shared_mock(self, put_doc_side_effect): - """ - Get a default class for mocking the shared DB - """ - class defaultMockSharedDB(object): - get_doc = Mock(return_value=None) - put_doc = Mock(side_effect=put_doc_side_effect) - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - def __call__(self): - return self - return defaultMockSharedDB - - def _soledad_instance(self, user=ADDRESS, passphrase=u'123', - prefix='', - secrets_path='secrets.json', - local_db_path='soledad.u1db', server_url='', - cert_file=None, secret_id=None, - shared_db_class=None): - - def _put_doc_side_effect(doc): - self._doc_put = doc - - if shared_db_class is not None: - MockSharedDB = shared_db_class - else: - MockSharedDB = self.get_default_shared_mock( - _put_doc_side_effect) - - Soledad._shared_db = MockSharedDB() - return Soledad( - user, - passphrase, - secrets_path=os.path.join( - self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, # Soledad will fail if not given an url. - cert_file=cert_file, - secret_id=secret_id, - defer_encryption=self.defer_sync_encryption) - - def assertGetEncryptedDoc( - self, db, doc_id, doc_rev, content, has_conflicts): - """ - Assert that the document in the database looks correct. - """ - exp_doc = self.make_document(doc_id, doc_rev, content, - has_conflicts=has_conflicts) - doc = db.get_doc(doc_id) - - if ENC_SCHEME_KEY in doc.content: - # XXX check for SYM_KEY too - key = self._soledad._crypto.doc_passphrase(doc.doc_id) - secret = self._soledad._crypto.secret - decrypted = decrypt_doc_dict( - doc.content, doc.doc_id, doc.rev, - key, secret) - doc.set_json(decrypted) - self.assertEqual(exp_doc.doc_id, doc.doc_id) - self.assertEqual(exp_doc.rev, doc.rev) - self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts) - self.assertEqual(exp_doc.content, doc.content) - - -# Key material for testing -KEY_FINGERPRINT = "E36E738D69173C13D709E44F2F455E2824D18DDF" -PUBLIC_KEY = """ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -mQINBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -tBxMZWFwIFRlc3QgS2V5IDxsZWFwQGxlYXAuc2U+iQI3BBMBCAAhBQJQvfnZAhsD -BQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEC9FXigk0Y3fT7EQAKH3IuRniOpb -T/DDIgwwjz3oxB/W0DDMyPXowlhSOuM0rgGfntBpBb3boezEXwL86NPQxNGGruF5 -hkmecSiuPSvOmQlqlS95NGQp6hNG0YaKColh+Q5NTspFXCAkFch9oqUje0LdxfSP -QfV9UpeEvGyPmk1I9EJV/YDmZ4+Djge1d7qhVZInz4Rx1NrSyF/Tc2EC0VpjQFsU -Y9Kb2YBBR7ivG6DBc8ty0jJXi7B4WjkFcUEJviQpMF2dCLdonCehYs1PqsN1N7j+ -eFjQd+hqVMJgYuSGKjvuAEfClM6MQw7+FmFwMyLgK/Ew/DttHEDCri77SPSkOGSI -txCzhTg6798f6mJr7WcXmHX1w1Vcib5FfZ8vTDFVhz/XgAgArdhPo9V6/1dgSSiB -KPQ/spsco6u5imdOhckERE0lnAYvVT6KE81TKuhF/b23u7x+Wdew6kK0EQhYA7wy -7LmlaNXc7rMBQJ9Z60CJ4JDtatBWZ0kNrt2VfdDHVdqBTOpl0CraNUjWE5YMDasr -K2dF5IX8D3uuYtpZnxqg0KzyLg0tzL0tvOL1C2iudgZUISZNPKbS0z0v+afuAAnx -2pTC3uezbh2Jt8SWTLhll4i0P4Ps5kZ6HQUO56O+/Z1cWovX+mQekYFmERySDR9n -3k1uAwLilJmRmepGmvYbB8HloV8HqwgguQINBFC9+dkBEAC0I/xn1uborMgDvBtf -H0sEhwnXBC849/32zic6udB6/3Efk9nzbSpL3FSOuXITZsZgCHPkKarnoQ2ztMcS -sh1ke1C5gQGms75UVmM/nS+2YI4vY8OX/GC/on2vUyncqdH+bR6xH5hx4NbWpfTs -iQHmz5C6zzS/kuabGdZyKRaZHt23WQ7JX/4zpjqbC99DjHcP9BSk7tJ8wI4bkMYD -uFVQdT9O6HwyKGYwUU4sAQRAj7XCTGvVbT0dpgJwH4RmrEtJoHAx4Whg8mJ710E0 -GCmzf2jqkNuOw76ivgk27Kge+Hw00jmJjQhHY0yVbiaoJwcRrPKzaSjEVNgrpgP3 -lXPRGQArgESsIOTeVVHQ8fhK2YtTeCY9rIiO+L0OX2xo9HK7hfHZZWL6rqymXdyS -fhzh/f6IPyHFWnvj7Brl7DR8heMikygcJqv+ed2yx7iLyCUJ10g12I48+aEj1aLe -dP7lna32iY8/Z0SHQLNH6PXO9SlPcq2aFUgKqE75A/0FMk7CunzU1OWr2ZtTLNO1 -WT/13LfOhhuEq9jTyTosn0WxBjJKq18lnhzCXlaw6EAtbA7CUwsD3CTPR56aAXFK -3I7KXOVAqggrvMe5Tpdg5drfYpI8hZovL5aAgb+7Y5ta10TcJdUhS5K3kFAWe/td -U0cmWUMDP1UMSQ5Jg6JIQVWhSwARAQABiQIfBBgBCAAJBQJQvfnZAhsMAAoJEC9F -Xigk0Y3fRwsP/i0ElYCyxeLpWJTwo1iCLkMKz2yX1lFVa9nT1BVTPOQwr/IAc5OX -NdtbJ14fUsKL5pWgW8OmrXtwZm1y4euI1RPWWubG01ouzwnGzv26UcuHeqC5orZj -cOnKtL40y8VGMm8LoicVkRJH8blPORCnaLjdOtmA3rx/v2EXrJpSa3AhOy0ZSRXk -ZSrK68AVNwamHRoBSYyo0AtaXnkPX4+tmO8X8BPfj125IljubvwZPIW9VWR9UqCE -VPfDR1XKegVb6VStIywF7kmrknM1C5qUY28rdZYWgKorw01hBGV4jTW0cqde3N51 -XT1jnIAa+NoXUM9uQoGYMiwrL7vNsLlyyiW5ayDyV92H/rIuiqhFgbJsHTlsm7I8 -oGheR784BagAA1NIKD1qEO9T6Kz9lzlDaeWS5AUKeXrb7ZJLI1TTCIZx5/DxjLqM -Tt/RFBpVo9geZQrvLUqLAMwdaUvDXC2c6DaCPXTh65oCZj/hqzlJHH+RoTWWzKI+ -BjXxgUWF9EmZUBrg68DSmI+9wuDFsjZ51BcqvJwxyfxtTaWhdoYqH/UQS+D1FP3/ -diZHHlzwVwPICzM9ooNTgbrcDzyxRkIVqsVwBq7EtzcvgYUyX53yG25Giy6YQaQ2 -ZtQ/VymwFL3XdUWV6B/hU4PVAFvO3qlOtdJ6TpE+nEWgcWjCv5g7RjXX -=MuOY ------END PGP PUBLIC KEY BLOCK----- -""" -PRIVATE_KEY = """ ------BEGIN PGP PRIVATE KEY BLOCK----- -Version: GnuPG v1.4.10 (GNU/Linux) - -lQcYBFC9+dkBEADNRfwV23TWEoGc/x0wWH1P7PlXt8MnC2Z1kKaKKmfnglVrpOiz -iLWoiU58sfZ0L5vHkzXHXCBf6Eiy/EtUIvdiWAn+yASJ1mk5jZTBKO/WMAHD8wTO -zpMsFmWyg3xc4DkmFa9KQ5EVU0o/nqPeyQxNMQN7px5pPwrJtJFmPxnxm+aDkPYx -irDmz/4DeDNqXliazGJKw7efqBdlwTHkl9Akw2gwy178pmsKwHHEMOBOFFvX61AT -huKqHYmlCGSliwbrJppTG7jc1/ls3itrK+CWTg4txREkSpEVmfcASvw/ZqLbjgfs -d/INMwXnR9U81O8+7LT6yw/ca4ppcFoJD7/XJbkRiML6+bJ4Dakiy6i727BzV17g -wI1zqNvm5rAhtALKfACha6YO43aJzairO4II1wxVHvRDHZn2IuKDDephQ3Ii7/vb -hUOf6XCSmchkAcpKXUOvbxm1yfB1LRa64mMc2RcZxf4mW7KQkulBsdV5QG2276lv -U2UUy2IutXcGP5nXC+f6sJJGJeEToKJ57yiO/VWJFjKN8SvP+7AYsQSqINUuEf6H -T5gCPCraGMkTUTPXrREvu7NOohU78q6zZNaL3GW8ai7eSeANSuQ8Vzffx7Wd8Y7i -Pw9sYj0SMFs1UgjbuL6pO5ueHh+qyumbtAq2K0Bci0kqOcU4E9fNtdiovQARAQAB -AA/+JHtlL39G1wsH9R6UEfUQJGXR9MiIiwZoKcnRB2o8+DS+OLjg0JOh8XehtuCs -E/8oGQKtQqa5bEIstX7IZoYmYFiUQi9LOzIblmp2vxOm+HKkxa4JszWci2/ZmC3t -KtaA4adl9XVnshoQ7pijuCMUKB3naBEOAxd8s9d/JeReGIYkJErdrnVfNk5N71Ds -FmH5Ll3XtEDvgBUQP3nkA6QFjpsaB94FHjL3gDwum/cxzj6pCglcvHOzEhfY0Ddb -J967FozQTaf2JW3O+w3LOqtcKWpq87B7+O61tVidQPSSuzPjCtFF0D2LC9R/Hpky -KTMQ6CaKja4MPhjwywd4QPcHGYSqjMpflvJqi+kYIt8psUK/YswWjnr3r4fbuqVY -VhtiHvnBHQjz135lUqWvEz4hM3Xpnxydx7aRlv5NlevK8+YIO5oFbWbGNTWsPZI5 -jpoFBpSsnR1Q5tnvtNHauvoWV+XN2qAOBTG+/nEbDYH6Ak3aaE9jrpTdYh0CotYF -q7csANsDy3JvkAzeU6WnYpsHHaAjqOGyiZGsLej1UcXPFMosE/aUo4WQhiS8Zx2c -zOVKOi/X5vQ2GdNT9Qolz8AriwzsvFR+bxPzyd8V6ALwDsoXvwEYinYBKK8j0OPv -OOihSR6HVsuP9NUZNU9ewiGzte/+/r6pNXHvR7wTQ8EWLcEIAN6Zyrb0bHZTIlxt -VWur/Ht2mIZrBaO50qmM5RD3T5oXzWXi/pjLrIpBMfeZR9DWfwQwjYzwqi7pxtYx -nJvbMuY505rfnMoYxb4J+cpRXV8MS7Dr1vjjLVUC9KiwSbM3gg6emfd2yuA93ihv -Pe3mffzLIiQa4mRE3wtGcioC43nWuV2K2e1KjxeFg07JhrezA/1Cak505ab/tmvP -4YmjR5c44+yL/YcQ3HdFgs4mV+nVbptRXvRcPpolJsgxPccGNdvHhsoR4gwXMS3F -RRPD2z6x8xeN73Q4KH3bm01swQdwFBZbWVfmUGLxvN7leCdfs9+iFJyqHiCIB6Iv -mQfp8F0IAOwSo8JhWN+V1dwML4EkIrM8wUb4yecNLkyR6TpPH/qXx4PxVMC+vy6x -sCtjeHIwKE+9vqnlhd5zOYh7qYXEJtYwdeDDmDbL8oks1LFfd+FyAuZXY33DLwn0 -cRYsr2OEZmaajqUB3NVmj3H4uJBN9+paFHyFSXrH68K1Fk2o3n+RSf2EiX+eICwI -L6rqoF5sSVUghBWdNegV7qfy4anwTQwrIMGjgU5S6PKW0Dr/3iO5z3qQpGPAj5OW -ATqPWkDICLbObPxD5cJlyyNE2wCA9VVc6/1d6w4EVwSq9h3/WTpATEreXXxTGptd -LNiTA1nmakBYNO2Iyo3djhaqBdWjk+EIAKtVEnJH9FAVwWOvaj1RoZMA5DnDMo7e -SnhrCXl8AL7Z1WInEaybasTJXn1uQ8xY52Ua4b8cbuEKRKzw/70NesFRoMLYoHTO -dyeszvhoDHberpGRTciVmpMu7Hyi33rM31K9epA4ib6QbbCHnxkWOZB+Bhgj1hJ8 -xb4RBYWiWpAYcg0+DAC3w9gfxQhtUlZPIbmbrBmrVkO2GVGUj8kH6k4UV6kUHEGY -HQWQR0HcbKcXW81ZXCCD0l7ROuEWQtTe5Jw7dJ4/QFuqZnPutXVRNOZqpl6eRShw -7X2/a29VXBpmHA95a88rSQsL+qm7Fb3prqRmuMCtrUZgFz7HLSTuUMR867QcTGVh -cCBUZXN0IEtleSA8bGVhcEBsZWFwLnNlPokCNwQTAQgAIQUCUL352QIbAwULCQgH -AwUVCgkICwUWAgMBAAIeAQIXgAAKCRAvRV4oJNGN30+xEACh9yLkZ4jqW0/wwyIM -MI896MQf1tAwzMj16MJYUjrjNK4Bn57QaQW926HsxF8C/OjT0MTRhq7heYZJnnEo -rj0rzpkJapUveTRkKeoTRtGGigqJYfkOTU7KRVwgJBXIfaKlI3tC3cX0j0H1fVKX -hLxsj5pNSPRCVf2A5mePg44HtXe6oVWSJ8+EcdTa0shf03NhAtFaY0BbFGPSm9mA -QUe4rxugwXPLctIyV4uweFo5BXFBCb4kKTBdnQi3aJwnoWLNT6rDdTe4/nhY0Hfo -alTCYGLkhio77gBHwpTOjEMO/hZhcDMi4CvxMPw7bRxAwq4u+0j0pDhkiLcQs4U4 -Ou/fH+pia+1nF5h19cNVXIm+RX2fL0wxVYc/14AIAK3YT6PVev9XYEkogSj0P7Kb -HKOruYpnToXJBERNJZwGL1U+ihPNUyroRf29t7u8flnXsOpCtBEIWAO8Muy5pWjV -3O6zAUCfWetAieCQ7WrQVmdJDa7dlX3Qx1XagUzqZdAq2jVI1hOWDA2rKytnReSF -/A97rmLaWZ8aoNCs8i4NLcy9Lbzi9QtornYGVCEmTTym0tM9L/mn7gAJ8dqUwt7n -s24dibfElky4ZZeItD+D7OZGeh0FDuejvv2dXFqL1/pkHpGBZhEckg0fZ95NbgMC -4pSZkZnqRpr2GwfB5aFfB6sIIJ0HGARQvfnZARAAtCP8Z9bm6KzIA7wbXx9LBIcJ -1wQvOPf99s4nOrnQev9xH5PZ820qS9xUjrlyE2bGYAhz5Cmq56ENs7THErIdZHtQ -uYEBprO+VFZjP50vtmCOL2PDl/xgv6J9r1Mp3KnR/m0esR+YceDW1qX07IkB5s+Q -us80v5LmmxnWcikWmR7dt1kOyV/+M6Y6mwvfQ4x3D/QUpO7SfMCOG5DGA7hVUHU/ -Tuh8MihmMFFOLAEEQI+1wkxr1W09HaYCcB+EZqxLSaBwMeFoYPJie9dBNBgps39o -6pDbjsO+or4JNuyoHvh8NNI5iY0IR2NMlW4mqCcHEazys2koxFTYK6YD95Vz0RkA -K4BErCDk3lVR0PH4StmLU3gmPayIjvi9Dl9saPRyu4Xx2WVi+q6spl3ckn4c4f3+ -iD8hxVp74+wa5ew0fIXjIpMoHCar/nndsse4i8glCddINdiOPPmhI9Wi3nT+5Z2t -9omPP2dEh0CzR+j1zvUpT3KtmhVICqhO+QP9BTJOwrp81NTlq9mbUyzTtVk/9dy3 -zoYbhKvY08k6LJ9FsQYySqtfJZ4cwl5WsOhALWwOwlMLA9wkz0eemgFxStyOylzl -QKoIK7zHuU6XYOXa32KSPIWaLy+WgIG/u2ObWtdE3CXVIUuSt5BQFnv7XVNHJllD -Az9VDEkOSYOiSEFVoUsAEQEAAQAP/1AagnZQZyzHDEgw4QELAspYHCWLXE5aZInX -wTUJhK31IgIXNn9bJ0hFiSpQR2xeMs9oYtRuPOu0P8oOFMn4/z374fkjZy8QVY3e -PlL+3EUeqYtkMwlGNmVw5a/NbNuNfm5Darb7pEfbYd1gPcni4MAYw7R2SG/57GbC -9gucvspHIfOSfBNLBthDzmK8xEKe1yD2eimfc2T7IRYb6hmkYfeds5GsqvGI6mwI -85h4uUHWRc5JOlhVM6yX8hSWx0L60Z3DZLChmc8maWnFXd7C8eQ6P1azJJbW71Ih -7CoK0XW4LE82vlQurSRFgTwfl7wFYszW2bOzCuhHDDtYnwH86Nsu0DC78ZVRnvxn -E8Ke/AJgrdhIOo4UAyR+aZD2+2mKd7/waOUTUrUtTzc7i8N3YXGi/EIaNReBXaq+ -ZNOp24BlFzRp+FCF/pptDW9HjPdiV09x0DgICmeZS4Gq/4vFFIahWctg52NGebT0 -Idxngjj+xDtLaZlLQoOz0n5ByjO/Wi0ANmMv1sMKCHhGvdaSws2/PbMR2r4caj8m -KXpIgdinM/wUzHJ5pZyF2U/qejsRj8Kw8KH/tfX4JCLhiaP/mgeTuWGDHeZQERAT -xPmRFHaLP9/ZhvGNh6okIYtrKjWTLGoXvKLHcrKNisBLSq+P2WeFrlme1vjvJMo/ -jPwLT5o9CADQmcbKZ+QQ1ZM9v99iDZol7SAMZX43JC019sx6GK0u6xouJBcLfeB4 -OXacTgmSYdTa9RM9fbfVpti01tJ84LV2SyL/VJq/enJF4XQPSynT/tFTn1PAor6o -tEAAd8fjKdJ6LnD5wb92SPHfQfXqI84rFEO8rUNIE/1ErT6DYifDzVCbfD2KZdoF -cOSp7TpD77sY1bs74ocBX5ejKtd+aH99D78bJSMM4pSDZsIEwnomkBHTziubPwJb -OwnATy0LmSMAWOw5rKbsh5nfwCiUTM20xp0t5JeXd+wPVWbpWqI2EnkCEN+RJr9i -7dp/ymDQ+Yt5wrsN3NwoyiexPOG91WQVCADdErHsnglVZZq9Z8Wx7KwecGCUurJ2 -H6lKudv5YOxPnAzqZS5HbpZd/nRTMZh2rdXCr5m2YOuewyYjvM757AkmUpM09zJX -MQ1S67/UX2y8/74TcRF97Ncx9HeELs92innBRXoFitnNguvcO6Esx4BTe1OdU6qR -ER3zAmVf22Le9ciXbu24DN4mleOH+OmBx7X2PqJSYW9GAMTsRB081R6EWKH7romQ -waxFrZ4DJzZ9ltyosEJn5F32StyLrFxpcrdLUoEaclZCv2qka7sZvi0EvovDVEBU -e10jOx9AOwf8Gj2ufhquQ6qgVYCzbP+YrodtkFrXRS3IsljIchj1M2ffB/0bfoUs -rtER9pLvYzCjBPg8IfGLw0o754Qbhh/ReplCRTusP/fQMybvCvfxreS3oyEriu/G -GufRomjewZ8EMHDIgUsLcYo2UHZsfF7tcazgxMGmMvazp4r8vpgrvW/8fIN/6Adu -tF+WjWDTvJLFJCe6O+BFJOWrssNrrra1zGtLC1s8s+Wfpe+bGPL5zpHeebGTwH1U -22eqgJArlEKxrfarz7W5+uHZJHSjF/K9ZvunLGD0n9GOPMpji3UO3zeM8IYoWn7E -/EWK1XbjnssNemeeTZ+sDh+qrD7BOi+vCX1IyBxbfqnQfJZvmcPWpruy1UsO+aIC -0GY8Jr3OL69dDQ21jueJAh8EGAEIAAkFAlC9+dkCGwwACgkQL0VeKCTRjd9HCw/+ -LQSVgLLF4ulYlPCjWIIuQwrPbJfWUVVr2dPUFVM85DCv8gBzk5c121snXh9Swovm -laBbw6ate3BmbXLh64jVE9Za5sbTWi7PCcbO/bpRy4d6oLmitmNw6cq0vjTLxUYy -bwuiJxWREkfxuU85EKdouN062YDevH+/YResmlJrcCE7LRlJFeRlKsrrwBU3BqYd -GgFJjKjQC1peeQ9fj62Y7xfwE9+PXbkiWO5u/Bk8hb1VZH1SoIRU98NHVcp6BVvp -VK0jLAXuSauSczULmpRjbyt1lhaAqivDTWEEZXiNNbRyp17c3nVdPWOcgBr42hdQ -z25CgZgyLCsvu82wuXLKJblrIPJX3Yf+si6KqEWBsmwdOWybsjygaF5HvzgFqAAD -U0goPWoQ71PorP2XOUNp5ZLkBQp5etvtkksjVNMIhnHn8PGMuoxO39EUGlWj2B5l -Cu8tSosAzB1pS8NcLZzoNoI9dOHrmgJmP+GrOUkcf5GhNZbMoj4GNfGBRYX0SZlQ -GuDrwNKYj73C4MWyNnnUFyq8nDHJ/G1NpaF2hiof9RBL4PUU/f92JkceXPBXA8gL -Mz2ig1OButwPPLFGQhWqxXAGrsS3Ny+BhTJfnfIbbkaLLphBpDZm1D9XKbAUvdd1 -RZXoH+FTg9UAW87eqU610npOkT6cRaBxaMK/mDtGNdc= -=JTFu ------END PGP PRIVATE KEY BLOCK----- -""" +import os def load_tests(): diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 10d6c136..d2aef9bb 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -20,134 +20,21 @@ Test ObjectStore and Couch backend bits. """ -import re -import copy -import shutil -from base64 import b64decode -from mock import Mock -from urlparse import urljoin +import simplejson as json + +from urlparse import urljoin from u1db import errors as u1db_errors from couchdb.client import Server -from leap.common.files import mkdir_p +from testscenarios import TestWithScenarios + +from leap.soledad.common import couch, errors from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_backends from leap.soledad.common.tests.u1db_tests import test_sync -from leap.soledad.common import couch, errors -import simplejson as json - - -#----------------------------------------------------------------------------- -# A wrapper for running couchdb locally. -#----------------------------------------------------------------------------- - -import re -import os -import tempfile -import subprocess -import time -import unittest - - -# from: https://github.com/smcq/paisley/blob/master/paisley/test/util.py -# TODO: include license of above project. -class CouchDBWrapper(object): - """ - Wrapper for external CouchDB instance which is started and stopped for - testing. - """ - - def start(self): - """ - Start a CouchDB instance for a test. - """ - self.tempdir = tempfile.mkdtemp(suffix='.couch.test') - - path = os.path.join(os.path.dirname(__file__), - 'couchdb.ini.template') - handle = open(path) - conf = handle.read() % { - 'tempdir': self.tempdir, - } - handle.close() - - confPath = os.path.join(self.tempdir, 'test.ini') - handle = open(confPath, 'w') - handle.write(conf) - handle.close() - - # create the dirs from the template - mkdir_p(os.path.join(self.tempdir, 'lib')) - mkdir_p(os.path.join(self.tempdir, 'log')) - args = ['couchdb', '-n', '-a', confPath] - null = open('/dev/null', 'w') - - self.process = subprocess.Popen( - args, env=None, stdout=null.fileno(), stderr=null.fileno(), - close_fds=True) - # find port - logPath = os.path.join(self.tempdir, 'log', 'couch.log') - while not os.path.exists(logPath): - if self.process.poll() is not None: - got_stdout, got_stderr = "", "" - if self.process.stdout is not None: - got_stdout = self.process.stdout.read() - - if self.process.stderr is not None: - got_stderr = self.process.stderr.read() - raise Exception(""" -couchdb exited with code %d. -stdout: -%s -stderr: -%s""" % ( - self.process.returncode, got_stdout, got_stderr)) - time.sleep(0.01) - while os.stat(logPath).st_size == 0: - time.sleep(0.01) - PORT_RE = re.compile( - 'Apache CouchDB has started on http://127.0.0.1:(?P\d+)') - - handle = open(logPath) - line = handle.read() - handle.close() - m = PORT_RE.search(line) - if not m: - self.stop() - raise Exception("Cannot find port in line %s" % line) - self.port = int(m.group('port')) - - def stop(self): - """ - Terminate the CouchDB instance. - """ - self.process.terminate() - self.process.communicate() - shutil.rmtree(self.tempdir) - - -class CouchDBTestCase(unittest.TestCase): - """ - TestCase base class for tests against a real CouchDB server. - """ - - @classmethod - def setUpClass(cls): - """ - Make sure we have a CouchDB instance for a test. - """ - cls.wrapper = CouchDBWrapper() - cls.wrapper.start() - #self.db = self.wrapper.db - - @classmethod - def tearDownClass(cls): - """ - Stop CouchDB instance for test. - """ - cls.wrapper.stop() +from leap.soledad.common.tests.util import CouchDBTestCase #----------------------------------------------------------------------------- @@ -239,7 +126,8 @@ COUCH_SCENARIOS = [ ] -class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): +class CouchTests( + TestWithScenarios, test_backends.AllDatabaseTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -262,7 +150,8 @@ class CouchTests(test_backends.AllDatabaseTests, CouchDBTestCase): test_backends.AllDatabaseTests.tearDown(self) -class CouchDatabaseTests(test_backends.LocalDatabaseTests, CouchDBTestCase): +class CouchDatabaseTests( + TestWithScenarios, test_backends.LocalDatabaseTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -271,7 +160,7 @@ class CouchDatabaseTests(test_backends.LocalDatabaseTests, CouchDBTestCase): test_backends.LocalDatabaseTests.tearDown(self) -class CouchValidateGenNTransIdTests( +class CouchValidateGenNTransIdTests(TestWithScenarios, test_backends.LocalDatabaseValidateGenNTransIdTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -281,7 +170,7 @@ class CouchValidateGenNTransIdTests( test_backends.LocalDatabaseValidateGenNTransIdTests.tearDown(self) -class CouchValidateSourceGenTests( +class CouchValidateSourceGenTests(TestWithScenarios, test_backends.LocalDatabaseValidateSourceGenTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -291,7 +180,7 @@ class CouchValidateSourceGenTests( test_backends.LocalDatabaseValidateSourceGenTests.tearDown(self) -class CouchWithConflictsTests( +class CouchWithConflictsTests(TestWithScenarios, test_backends.LocalDatabaseWithConflictsTests, CouchDBTestCase): scenarios = COUCH_SCENARIOS @@ -325,23 +214,11 @@ simple_doc = tests.simple_doc nested_doc = tests.nested_doc -class CouchDatabaseSyncTargetTests(test_sync.DatabaseSyncTargetTests, - CouchDBTestCase): +class CouchDatabaseSyncTargetTests( + TestWithScenarios, test_sync.DatabaseSyncTargetTests, CouchDBTestCase): scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios)) - def setUp(self): - # we implement parents' setUp methods here to prevent from launching - # more couch instances then needed. - tests.TestCase.setUp(self) - self.server = self.server_thread = None - self.db, self.st = self.create_db_and_target(self) - self.other_changes = [] - - def tearDown(self): - self.db.delete_database() - test_sync.DatabaseSyncTargetTests.tearDown(self) - def test_sync_exchange_returns_many_new_docs(self): # This test was replicated to allow dictionaries to be compared after # JSON expansion (because one dictionary may have many different @@ -372,7 +249,7 @@ from u1db.backends.inmemory import InMemoryIndex class IndexedCouchDatabase(couch.CouchDatabase): def __init__(self, url, dbname, replica_uid=None, ensure_ddocs=True): - old_class.__init__(self, url, dbname, replica_uid=replica_uid, + old_class.__init__(self, url, dbname, replica_uid=replica_uid, ensure_ddocs=ensure_ddocs) self._indexes = {} @@ -458,7 +335,8 @@ for name, scenario in COUCH_SCENARIOS: scenario = dict(scenario) -class CouchDatabaseSyncTests(test_sync.DatabaseSyncTests, CouchDBTestCase): +class CouchDatabaseSyncTests( + TestWithScenarios, test_sync.DatabaseSyncTests, CouchDBTestCase): scenarios = sync_scenarios @@ -498,6 +376,7 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): def tearDown(self): self.db.delete_database() self.db.close() + CouchDBTestCase.tearDown(self) def test_missing_design_doc_raises(self): """ @@ -670,6 +549,3 @@ class CouchDatabaseExceptionsTests(CouchDBTestCase): self.assertRaises( errors.MissingDesignDocDeletedError, self.db._do_set_replica_gen_and_trans_id, 1, 2, 3) - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py index 6465eb80..83cee469 100644 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py @@ -15,26 +15,25 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ -Test atomocity for couch operations. +Test atomicity of couch operations. """ import os -import mock import tempfile import threading - from urlparse import urljoin - +from twisted.internet import defer from leap.soledad.client import Soledad from leap.soledad.common.couch import CouchDatabase, CouchServerState -from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer -from leap.soledad.common.tests.test_sync_target import ( + +from leap.soledad.common.tests.util import ( make_token_soledad_app, - make_leap_document_for_test, - token_leap_sync_target, + make_soledad_document_for_test, + token_soledad_sync_target, ) +from leap.soledad.common.tests.test_couch import CouchDBTestCase +from leap.soledad.common.tests.u1db_tests import TestCaseWithServer from leap.soledad.common.tests.test_server import _couch_ensure_database @@ -52,15 +51,15 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): def make_app_after_state(state): return make_token_soledad_app(state) - make_document_for_test = make_leap_document_for_test + make_document_for_test = make_soledad_document_for_test - sync_target = token_leap_sync_target + sync_target = token_soledad_sync_target def _soledad_instance(self, user='user-uuid', passphrase=u'123', prefix='', - secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, + secrets_path='secrets.json', local_db_path='soledad.u1db', server_url='', - cert_file=None, auth_token=None, secret_id=None): + cert_file=None, auth_token=None): """ Instantiate Soledad. """ @@ -70,19 +69,6 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): def _put_doc_side_effect(doc): self._doc_put = doc - # we need a mocked shared db or else Soledad will try to access the - # network to find if there are uploaded secrets. - class MockSharedDB(object): - - get_doc = mock.Mock(return_value=None) - put_doc = mock.Mock(side_effect=_put_doc_side_effect) - lock = mock.Mock(return_value=('atoken', 300)) - unlock = mock.Mock() - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() return Soledad( user, passphrase, @@ -92,7 +78,7 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): server_url=server_url, cert_file=cert_file, auth_token=auth_token, - secret_id=secret_id) + shared_db=self.get_default_shared_mock(_put_doc_side_effect)) def make_app(self): self.request_state = CouchServerState(self._couch_url, 'shared', @@ -126,7 +112,6 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): puts. """ doc = self.db.create_doc({'ops': 0}) - ops = 1 docs = [doc.doc_id] for i in range(0, REPEAT_TIMES): self.assertEqual( @@ -183,24 +168,27 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): auth_token='auth-token', server_url=self.getURL()) - def _create_docs_and_sync(sol, syncs): - # create a lot of documents - for i in range(0, REPEAT_TIMES): - sol.create_doc({}) + def _create_docs(results): + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + deferreds.append(sol.create_doc({})) + return defer.DeferredList(deferreds) + + def _assert_transaction_and_sync_logs(results, sync_idx): # assert sizes of transaction and sync logs self.assertEqual( - syncs*REPEAT_TIMES, + sync_idx*REPEAT_TIMES, len(self.db._get_transaction_log())) self.assertEqual( - 1 if syncs > 0 else 0, + 1 if sync_idx > 0 else 0, len(self.db._database.view('syncs/log').rows)) - # sync to the remote db - sol.sync() - gen, docs = self.db.get_all_docs() - self.assertEqual((syncs+1)*REPEAT_TIMES, gen) - self.assertEqual((syncs+1)*REPEAT_TIMES, len(docs)) + + def _assert_sync(results, sync_idx): + gen, docs = results + self.assertEqual((sync_idx+1)*REPEAT_TIMES, gen) + self.assertEqual((sync_idx+1)*REPEAT_TIMES, len(docs)) # assert sizes of transaction and sync logs - self.assertEqual((syncs+1)*REPEAT_TIMES, + self.assertEqual((sync_idx+1)*REPEAT_TIMES, len(self.db._get_transaction_log())) sync_log_rows = self.db._database.view('syncs/log').rows sync_log = sync_log_rows[0].value @@ -210,14 +198,32 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): # assert sync_log has exactly 1 row self.assertEqual(1, len(sync_log_rows)) # assert it has the correct replica_uid, gen and trans_id - self.assertEqual(sol._db._replica_uid, replica_uid) - sol_gen, sol_trans_id = sol._db._get_generation_info() + self.assertEqual(sol._dbpool.replica_uid, replica_uid) + conn_key = sol._dbpool._u1dbconnections.keys().pop() + conn = sol._dbpool._u1dbconnections[conn_key] + sol_gen, sol_trans_id = conn._get_generation_info() self.assertEqual(sol_gen, known_gen) self.assertEqual(sol_trans_id, known_trans_id) + + # create some documents + d = _create_docs(None) - _create_docs_and_sync(sol, 0) - _create_docs_and_sync(sol, 1) - sol.close() + # sync first time and assert success + d.addCallback(_assert_transaction_and_sync_logs, 0) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: sol.get_all_docs()) + d.addCallback(_assert_sync, 0) + + # create more docs, sync second time and assert success + d.addCallback(_create_docs) + d.addCallback(_assert_transaction_and_sync_logs, 1) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: sol.get_all_docs()) + d.addCallback(_assert_sync, 1) + + d.addCallback(lambda _: sol.close()) + + return d # # Concurrency tests @@ -313,86 +319,76 @@ class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): """ Assert that the sync_log is correct after concurrent syncs. """ - threads = [] docs = [] - pool = threading.BoundedSemaphore(value=1) + self.startServer() + sol = self._soledad_instance( auth_token='auth-token', server_url=self.getURL()) - def _run_method(self): - # create a lot of documents - doc = self._params['sol'].create_doc({}) - pool.acquire() - docs.append(doc.doc_id) - pool.release() + def _save_doc_ids(results): + for doc in results: + docs.append(doc.doc_id) - # launch threads to create documents in parallel + # create documents in parallel + deferreds = [] for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread( - {'sol': sol, 'syncs': i}, - _run_method) - thread.start() - threads.append(thread) + d = sol.create_doc({}) + deferreds.append(d) - # wait for threads to finish - for thread in threads: - thread.join() + # wait for documents creation and sync + d = defer.gatherResults(deferreds) + d.addCallback(_save_doc_ids) + d.addCallback(lambda _: sol.sync()) - # do the sync! - sol.sync() + def _assert_logs(results): + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - sol.close() + d.addCallback(_assert_logs) + d.addCallback(lambda _: sol.close()) + + return d def test_concurrent_syncs_do_not_fail(self): """ Assert that concurrent attempts to sync end up being executed sequentially and do not fail. """ - threads = [] docs = [] - pool = threading.BoundedSemaphore(value=1) + self.startServer() + sol = self._soledad_instance( auth_token='auth-token', server_url=self.getURL()) - def _run_method(self): - # create a lot of documents - doc = self._params['sol'].create_doc({}) - # do the sync! - sol.sync() - pool.acquire() - docs.append(doc.doc_id) - pool.release() - - # launch threads to create documents in parallel - for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread( - {'sol': sol, 'syncs': i}, - _run_method) - thread.start() - threads.append(thread) - - # wait for threads to finish - for thread in threads: - thread.join() - - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - sol.close() + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + d = sol.create_doc({}) + d.addCallback(lambda doc: docs.append(doc.doc_id)) + d.addCallback(lambda _: sol.sync()) + deferreds.append(d) + + def _assert_logs(results): + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + d = defer.gatherResults(deferreds) + d.addCallback(_assert_logs) + d.addCallback(lambda _: sol.close()) + + return d diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index f5fb4b7a..fdad8aac 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -23,7 +23,7 @@ import binascii from leap.soledad.client import crypto from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.tests import BaseSoledadTest +from leap.soledad.common.tests.util import BaseSoledadTest from leap.soledad.common.crypto import WrongMacError from leap.soledad.common.crypto import UnknownMacMethodError from leap.soledad.common.crypto import EncryptionMethods @@ -82,7 +82,7 @@ class RecoveryDocumentTestCase(BaseSoledadTest): rd = self._soledad.secrets._export_recovery_document() s = self._soledad_instance() s.secrets._import_recovery_document(rd) - s.set_secret_id(self._soledad.secrets._secret_id) + s.secrets.set_secret_id(self._soledad.secrets._secret_id) self.assertEqual(self._soledad.storage_secret, s.storage_secret, 'Failed settinng secret for symmetric encryption.') @@ -95,7 +95,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): # instantiate and save secret_id sol = self._soledad_instance(user='user@leap.se') self.assertTrue(len(sol.secrets._secrets) == 1) - secret_id_1 = sol.secret_id + secret_id_1 = sol.secrets.secret_id # assert id is hash of secret self.assertTrue( secret_id_1 == hashlib.sha256(sol.storage_secret).hexdigest()) @@ -104,9 +104,8 @@ class SoledadSecretsTestCase(BaseSoledadTest): self.assertTrue(secret_id_1 != secret_id_2) sol.close() # re-instantiate - sol = self._soledad_instance( - user='user@leap.se', - secret_id=secret_id_1) + sol = self._soledad_instance(user='user@leap.se') + sol.secrets.set_secret_id(secret_id_1) # assert ids are valid self.assertTrue(len(sol.secrets._secrets) == 2) self.assertTrue(secret_id_1 in sol.secrets._secrets) @@ -117,7 +116,7 @@ class SoledadSecretsTestCase(BaseSoledadTest): secret_length = sol.secrets.GEN_SECRET_LENGTH self.assertTrue(len(sol.storage_secret) == secret_length) # assert format of secret 2 - sol.set_secret_id(secret_id_2) + sol.secrets.set_secret_id(secret_id_2) self.assertTrue(sol.storage_secret is not None) self.assertIsInstance(sol.storage_secret, str) self.assertTrue(len(sol.storage_secret) == secret_length) @@ -134,12 +133,12 @@ class SoledadSecretsTestCase(BaseSoledadTest): "Should have a secret at this point") # setting secret id to None should not interfere in the fact we have a # secret. - sol.set_secret_id(None) + sol.secrets.set_secret_id(None) self.assertTrue( sol.secrets._has_secret(), "Should have a secret at this point") # but not being able to decrypt correctly should - sol.secrets._secrets[sol.secret_id] = None + sol.secrets._secrets[sol.secrets.secret_id] = None self.assertFalse(sol.secrets._has_secret()) sol.close() diff --git a/common/src/leap/soledad/common/tests/test_http.py b/common/src/leap/soledad/common/tests/test_http.py index d21470e0..1f661b77 100644 --- a/common/src/leap/soledad/common/tests/test_http.py +++ b/common/src/leap/soledad/common/tests/test_http.py @@ -20,8 +20,6 @@ Test Leap backend bits: test http database from u1db.remote import http_database from leap.soledad.client import auth - -from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_http_database @@ -59,6 +57,3 @@ class TestHTTPDatabaseWithCreds( 'token': 'auth-token', }}) self.assertIn('token', db1._creds) - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_http_client.py b/common/src/leap/soledad/common/tests/test_http_client.py index 3169398b..db731c32 100644 --- a/common/src/leap/soledad/common/tests/test_http_client.py +++ b/common/src/leap/soledad/common/tests/test_http_client.py @@ -21,8 +21,9 @@ import json from u1db.remote import http_client +from testscenarios import TestWithScenarios + from leap.soledad.client import auth -from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_http_client from leap.soledad.server.auth import SoledadTokenAuthMiddleware @@ -31,7 +32,9 @@ from leap.soledad.server.auth import SoledadTokenAuthMiddleware # The following tests come from `u1db.tests.test_http_client`. #----------------------------------------------------------------------------- -class TestSoledadClientBase(test_http_client.TestHTTPClientBase): +class TestSoledadClientBase( + TestWithScenarios, + test_http_client.TestHTTPClientBase): """ This class should be used to test Token auth. """ @@ -90,7 +93,7 @@ class TestSoledadClientBase(test_http_client.TestHTTPClientBase): "message": e.message})] uuid, token = encoded.decode('base64').split(':', 1) if uuid != 'user-uuid' and token != 'auth-token': - return unauth_err("Incorrect address or token.") + return Exception("Incorrect address or token.") start_response("200 OK", [('Content-Type', 'application/json')]) return [json.dumps([environ['PATH_INFO'], uuid, token])] @@ -112,5 +115,3 @@ class TestSoledadClientBase(test_http_client.TestHTTPClientBase): res, headers = cli._request('GET', ['doc', 'token']) self.assertEqual( ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_https.py b/common/src/leap/soledad/common/tests/test_https.py index b6288188..4dd55754 100644 --- a/common/src/leap/soledad/common/tests/test_https.py +++ b/common/src/leap/soledad/common/tests/test_https.py @@ -14,30 +14,35 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + + """ Test Leap backend bits: https """ -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests import test_sync_target as test_st -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import test_https -from leap.soledad import client -from leap.soledad.server import SoledadApp from u1db.remote import http_client +from leap.soledad import client + +from testscenarios import TestWithScenarios + +from leap.soledad.common.tests.u1db_tests import test_backends +from leap.soledad.common.tests.u1db_tests import test_https +from leap.soledad.common.tests.util import ( + BaseSoledadTest, + make_soledad_document_for_test, + make_soledad_app, + make_token_soledad_app, +) -def make_soledad_app(state): - return SoledadApp(state) LEAP_SCENARIOS = [ ('http', { 'make_database_for_test': test_backends.make_http_database_for_test, 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': test_st.make_leap_document_for_test, - 'make_app_with_state': test_st.make_soledad_app}), + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_soledad_app}), ] @@ -55,14 +60,15 @@ def token_leap_https_sync_target(test, host, path): class TestSoledadSyncTargetHttpsSupport( + TestWithScenarios, test_https.TestHttpSyncTargetHttpsSupport, BaseSoledadTest): scenarios = [ ('token_soledad_https', {'server_def': test_https.https_server_def, - 'make_app_with_state': test_st.make_token_soledad_app, - 'make_document_for_test': test_st.make_leap_document_for_test, + 'make_app_with_state': make_token_soledad_app, + 'make_document_for_test': make_soledad_document_for_test, 'sync_target': token_leap_https_sync_target}), ] @@ -71,8 +77,8 @@ class TestSoledadSyncTargetHttpsSupport( # run smoothly with standard u1db. test_https.TestHttpSyncTargetHttpsSupport.setUp(self) # so here monkey patch again to test our functionality. - http_client._VerifiedHTTPSConnection = client.VerifiedHTTPSConnection - client.SOLEDAD_CERT = http_client.CA_CERTS + http_client._VerifiedHTTPSConnection = client.api.VerifiedHTTPSConnection + client.api.SOLEDAD_CERT = http_client.CA_CERTS def test_working(self): """ @@ -83,7 +89,7 @@ class TestSoledadSyncTargetHttpsSupport( """ self.startServer() db = self.request_state._create_database('test') - self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) + self.patch(client.api, 'SOLEDAD_CERT', self.cacert_pem) remote_target = self.getSyncTarget('localhost', 'test') remote_target.record_sync_info('other-id', 2, 'T-id') self.assertEqual( @@ -99,10 +105,8 @@ class TestSoledadSyncTargetHttpsSupport( """ self.startServer() self.request_state._create_database('test') - self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) + self.patch(client.api, 'SOLEDAD_CERT', self.cacert_pem) remote_target = self.getSyncTarget('127.0.0.1', 'test') self.assertRaises( http_client.CertificateError, remote_target.record_sync_info, 'other-id', 2, 'T-id') - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index acd0a54c..836bd74a 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -19,29 +19,28 @@ Tests for server-related functionality. """ import os import tempfile -import simplejson as json import mock import time import binascii from urlparse import urljoin +from twisted.internet import defer -from leap.common.testing.basetest import BaseLeapTest from leap.soledad.common.couch import ( CouchServerState, CouchDatabase, ) -from leap.soledad.common.tests.u1db_tests import ( - TestCaseWithServer, - simple_doc, -) +from leap.soledad.common.tests.u1db_tests import TestCaseWithServer from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.test_target_soledad import ( +from leap.soledad.common.tests.util import ( make_token_soledad_app, - make_leap_document_for_test, + make_soledad_document_for_test, + token_soledad_sync_target, + BaseSoledadTest, ) -from leap.soledad.common.tests.test_sync_target import token_leap_sync_target -from leap.soledad.client import Soledad, crypto + +from leap.soledad.common import crypto +from leap.soledad.client import Soledad from leap.soledad.server import LockResource from leap.soledad.server.auth import URLToAuthorization @@ -58,7 +57,7 @@ def _couch_ensure_database(self, dbname): CouchServerState.ensure_database = _couch_ensure_database -class ServerAuthorizationTestCase(BaseLeapTest): +class ServerAuthorizationTestCase(BaseSoledadTest): """ Tests related to Soledad server authorization. """ @@ -272,19 +271,24 @@ class EncryptedSyncTestCase( Tests for encrypted sync using Soledad server backed by a couch database. """ + # increase twisted.trial's timeout because large files syncing might take + # some time to finish. + timeout = 500 + @staticmethod def make_app_with_state(state): return make_token_soledad_app(state) - make_document_for_test = make_leap_document_for_test + make_document_for_test = make_soledad_document_for_test - sync_target = token_leap_sync_target + sync_target = token_soledad_sync_target def _soledad_instance(self, user='user-uuid', passphrase=u'123', prefix='', - secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, - local_db_path='soledad.u1db', server_url='', - cert_file=None, auth_token=None, secret_id=None): + secrets_path='secrets.json', + local_db_path='soledad.u1db', + server_url='', + cert_file=None, auth_token=None): """ Instantiate Soledad. """ @@ -294,20 +298,15 @@ class EncryptedSyncTestCase( def _put_doc_side_effect(doc): self._doc_put = doc - # we need a mocked shared db or else Soledad will try to access the - # network to find if there are uploaded secrets. - class MockSharedDB(object): - - get_doc = mock.Mock(return_value=None) - put_doc = mock.Mock(side_effect=_put_doc_side_effect) - lock = mock.Mock(return_value=('atoken', 300)) - unlock = mock.Mock() - close = mock.Mock() - - def __call__(self): - return self + if not server_url: + # attempt to find the soledad server url + server_address = None + server = getattr(self, 'server', None) + if server: + server_address = getattr(self.server, 'server_address', None) + if server_address: + server_url = 'http://%s:%d' % (server_address) - Soledad._shared_db = MockSharedDB() return Soledad( user, passphrase, @@ -317,7 +316,7 @@ class EncryptedSyncTestCase( server_url=server_url, cert_file=cert_file, auth_token=auth_token, - secret_id=secret_id) + shared_db=self.get_default_shared_mock(_put_doc_side_effect)) def make_app(self): self.request_state = CouchServerState(self._couch_url, 'shared', @@ -325,70 +324,122 @@ class EncryptedSyncTestCase( return self.make_app_with_state(self.request_state) def setUp(self): - TestCaseWithServer.setUp(self) + # the order of the following initializations is crucial because of + # dependencies. + # XXX explain better CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") self._couch_url = 'http://localhost:' + str(self.wrapper.port) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + TestCaseWithServer.setUp(self) def tearDown(self): CouchDBTestCase.tearDown(self) TestCaseWithServer.tearDown(self) - def test_encrypted_sym_sync(self): + def _test_encrypted_sym_sync(self, passphrase=u'123', doc_size=2, + number_of_docs=1): """ Test the complete syncing chain between two soledad dbs using a Soledad server backed by a couch database. """ self.startServer() + # instantiate soledad and create a document sol1 = self._soledad_instance( # token is verified in test_target.make_token_soledad_app - auth_token='auth-token' - ) - _, doclist = sol1.get_all_docs() - self.assertEqual([], doclist) - doc1 = sol1.create_doc(json.loads(simple_doc)) + auth_token='auth-token', + passphrase=passphrase) + + # instantiate another soledad using the same secret as the previous + # one (so we can correctly verify the mac of the synced document) + sol2 = self._soledad_instance( + prefix='x', + auth_token='auth-token', + secrets_path=sol1._secrets_path, + passphrase=passphrase) + # ensure remote db exists before syncing db = CouchDatabase.open_database( urljoin(self._couch_url, 'user-user-uuid'), create=True, ensure_ddocs=True) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # assert doc was sent to couch db - _, doclist = db.get_all_docs() - self.assertEqual(1, len(doclist)) - couchdoc = doclist[0] - # assert document structure in couch server - self.assertEqual(doc1.doc_id, couchdoc.doc_id) - self.assertEqual(doc1.rev, couchdoc.rev) - self.assertEqual(6, len(couchdoc.content)) - self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content) - # instantiate soledad with empty db, but with same secrets path - sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') - _, doclist = sol2.get_all_docs() - self.assertEqual([], doclist) - sol2.secrets_path = sol1.secrets_path - sol2.secrets._load_secrets() - sol2.set_secret_id(sol1.secret_id) - # sync the new instance - sol2._server_url = self.getURL() - sol2.sync() - _, doclist = sol2.get_all_docs() - self.assertEqual(1, len(doclist)) - doc2 = doclist[0] - # assert incoming doc is equal to the first sent doc - self.assertEqual(doc1, doc2) - db.delete_database() - db.close() - sol1.close() - sol2.close() + + def _db1AssertEmptyDocList(results): + _, doclist = results + self.assertEqual([], doclist) + + def _db1CreateDocs(results): + deferreds = [] + for i in xrange(number_of_docs): + content = binascii.hexlify(os.urandom(doc_size/2)) + deferreds.append(sol1.create_doc({'data': content})) + return defer.DeferredList(deferreds) + + def _db1AssertDocsSyncedToServer(results): + _, sol_doclist = results + self.assertEqual(number_of_docs, len(sol_doclist)) + # assert doc was sent to couch db + _, couch_doclist = db.get_all_docs() + self.assertEqual(number_of_docs, len(couch_doclist)) + for i in xrange(number_of_docs): + soldoc = sol_doclist.pop() + couchdoc = couch_doclist.pop() + # assert document structure in couch server + self.assertEqual(soldoc.doc_id, couchdoc.doc_id) + self.assertEqual(soldoc.rev, couchdoc.rev) + self.assertEqual(6, len(couchdoc.content)) + self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content) + self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content) + self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content) + self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content) + self.assertTrue(crypto.MAC_KEY in couchdoc.content) + self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content) + + d = sol1.get_all_docs() + d.addCallback(_db1AssertEmptyDocList) + d.addCallback(_db1CreateDocs) + d.addCallback(lambda _: sol1.sync()) + d.addCallback(lambda _: sol1.get_all_docs()) + d.addCallback(_db1AssertDocsSyncedToServer) + + def _db2AssertEmptyDocList(results): + _, doclist = results + self.assertEqual([], doclist) + + def _getAllDocsFromBothDbs(results): + d1 = sol1.get_all_docs() + d2 = sol2.get_all_docs() + return defer.DeferredList([d1, d2]) + + d.addCallback(lambda _: sol2.get_all_docs()) + d.addCallback(_db2AssertEmptyDocList) + d.addCallback(lambda _: sol2.sync()) + d.addCallback(_getAllDocsFromBothDbs) + + def _assertDocSyncedFromDb1ToDb2(results): + r1, r2 = results + _, (gen1, doclist1) = r1 + _, (gen2, doclist2) = r2 + self.assertEqual(number_of_docs, gen1) + self.assertEqual(number_of_docs, gen2) + self.assertEqual(number_of_docs, len(doclist1)) + self.assertEqual(number_of_docs, len(doclist2)) + self.assertEqual(doclist1[0], doclist2[0]) + + d.addCallback(_assertDocSyncedFromDb1ToDb2) + + def _cleanUp(results): + db.delete_database() + db.close() + sol1.close() + sol2.close() + + d.addCallback(_cleanUp) + + return d + + def test_encrypted_sym_sync(self): + return self._test_encrypted_sym_sync() def test_encrypted_sym_sync_with_unicode_passphrase(self): """ @@ -396,152 +447,20 @@ class EncryptedSyncTestCase( Soledad server backed by a couch database, using an unicode passphrase. """ - self.startServer() - # instantiate soledad and create a document - sol1 = self._soledad_instance( - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token', - passphrase=u'ãáàäéàëíìïóòöõúùüñç', - ) - _, doclist = sol1.get_all_docs() - self.assertEqual([], doclist) - doc1 = sol1.create_doc(json.loads(simple_doc)) - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self._couch_url, 'user-user-uuid'), - create=True, - ensure_ddocs=True) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # assert doc was sent to couch db - _, doclist = db.get_all_docs() - self.assertEqual(1, len(doclist)) - couchdoc = doclist[0] - # assert document structure in couch server - self.assertEqual(doc1.doc_id, couchdoc.doc_id) - self.assertEqual(doc1.rev, couchdoc.rev) - self.assertEqual(6, len(couchdoc.content)) - self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content) - # instantiate soledad with empty db, but with same secrets path - sol2 = self._soledad_instance( - prefix='x', - auth_token='auth-token', - passphrase=u'ãáàäéàëíìïóòöõúùüñç', - ) - _, doclist = sol2.get_all_docs() - self.assertEqual([], doclist) - sol2.secrets_path = sol1.secrets_path - sol2.secrets._load_secrets() - sol2.set_secret_id(sol1.secret_id) - # sync the new instance - sol2._server_url = self.getURL() - sol2.sync() - _, doclist = sol2.get_all_docs() - self.assertEqual(1, len(doclist)) - doc2 = doclist[0] - # assert incoming doc is equal to the first sent doc - self.assertEqual(doc1, doc2) - db.delete_database() - db.close() - sol1.close() - sol2.close() + return self._test_encrypted_sym_sync(passphrase=u'ãáàäéàëíìïóòöõúùüñç') def test_sync_very_large_files(self): """ Test if Soledad can sync very large files. """ - # define the size of the "very large file" length = 100*(10**6) # 100 MB - self.startServer() - # instantiate soledad and create a document - sol1 = self._soledad_instance( - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token' - ) - _, doclist = sol1.get_all_docs() - self.assertEqual([], doclist) - content = binascii.hexlify(os.urandom(length/2)) # len() == length - doc1 = sol1.create_doc({'data': content}) - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self._couch_url, 'user-user-uuid'), - create=True, - ensure_ddocs=True) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # instantiate soledad with empty db, but with same secrets path - sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') - _, doclist = sol2.get_all_docs() - self.assertEqual([], doclist) - sol2.secrets_path = sol1.secrets_path - sol2.secrets._load_secrets() - sol2.set_secret_id(sol1.secret_id) - # sync the new instance - sol2._server_url = self.getURL() - sol2.sync() - _, doclist = sol2.get_all_docs() - self.assertEqual(1, len(doclist)) - doc2 = doclist[0] - # assert incoming doc is equal to the first sent doc - self.assertEqual(doc1, doc2) - # delete remote database - db.delete_database() - db.close() - sol1.close() - sol2.close() + return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1) def test_sync_many_small_files(self): """ Test if Soledad can sync many smallfiles. """ - number_of_docs = 100 - self.startServer() - # instantiate soledad and create a document - sol1 = self._soledad_instance( - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token' - ) - _, doclist = sol1.get_all_docs() - self.assertEqual([], doclist) - # create many small files - for i in range(0, number_of_docs): - sol1.create_doc(json.loads(simple_doc)) - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self._couch_url, 'user-user-uuid'), - create=True, - ensure_ddocs=True) - # sync with server - sol1._server_url = self.getURL() - sol1.sync() - # instantiate soledad with empty db, but with same secrets path - sol2 = self._soledad_instance(prefix='x', auth_token='auth-token') - _, doclist = sol2.get_all_docs() - self.assertEqual([], doclist) - sol2.secrets_path = sol1.secrets_path - sol2.secrets._load_secrets() - sol2.set_secret_id(sol1.secret_id) - # sync the new instance - sol2._server_url = self.getURL() - sol2.sync() - _, doclist = sol2.get_all_docs() - self.assertEqual(number_of_docs, len(doclist)) - # assert incoming docs are equal to sent docs - for doc in doclist: - self.assertEqual(sol1.get_doc(doc.doc_id), doc) - # delete remote database - db.delete_database() - db.close() - sol1.close() - sol2.close() - + return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100) class LockResourceTestCase( CouchDBTestCase, TestCaseWithServer): @@ -553,15 +472,18 @@ class LockResourceTestCase( def make_app_with_state(state): return make_token_soledad_app(state) - make_document_for_test = make_leap_document_for_test + make_document_for_test = make_soledad_document_for_test - sync_target = token_leap_sync_target + sync_target = token_soledad_sync_target def setUp(self): - TestCaseWithServer.setUp(self) + # the order of the following initializations is crucial because of + # dependencies. + # XXX explain better CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") self._couch_url = 'http://localhost:' + str(self.wrapper.port) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + TestCaseWithServer.setUp(self) # create the databases CouchDatabase.open_database( urljoin(self._couch_url, 'shared'), @@ -575,14 +497,14 @@ class LockResourceTestCase( self._couch_url, 'shared', 'tokens') def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) # delete remote database db = CouchDatabase.open_database( urljoin(self._couch_url, 'shared'), create=True, ensure_ddocs=True) db.delete_database() + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) def test__try_obtain_filesystem_lock(self): responder = mock.Mock() diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 31c02fc4..0b49d9f5 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -20,9 +20,8 @@ Tests for general Soledad functionality. import os from mock import Mock - from leap.common.events import events_pb2 as proto -from leap.soledad.common.tests import ( +from leap.soledad.common.tests.util import ( BaseSoledadTest, ADDRESS, ) @@ -30,10 +29,9 @@ from leap import soledad from leap.soledad.common.document import SoledadDocument from leap.soledad.common.crypto import WrongMacError from leap.soledad.client import Soledad -from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.adbapi import U1DBConnectionPool from leap.soledad.client.secrets import PassphraseTooShort from leap.soledad.client.shared_db import SoledadSharedDatabase -from leap.soledad.client.target import SoledadSyncTarget class AuxMethodsTestCase(BaseSoledadTest): @@ -41,18 +39,24 @@ class AuxMethodsTestCase(BaseSoledadTest): def test__init_dirs(self): sol = self._soledad_instance(prefix='_init_dirs') local_db_dir = os.path.dirname(sol.local_db_path) - secrets_path = os.path.dirname(sol.secrets_path) + secrets_path = os.path.dirname(sol.secrets.secrets_path) self.assertTrue(os.path.isdir(local_db_dir)) self.assertTrue(os.path.isdir(secrets_path)) - sol.close() - def test__init_db(self): + def _close_soledad(results): + sol.close() + + d = sol.create_doc({}) + d.addCallback(_close_soledad) + return d + + def test__init_u1db_sqlcipher_backend(self): sol = self._soledad_instance(prefix='_init_db') - self.assertIsInstance(sol._db, SQLCipherDatabase) + self.assertIsInstance(sol._dbpool, U1DBConnectionPool) self.assertTrue(os.path.isfile(sol.local_db_path)) sol.close() - def test__init_config_defaults(self): + def test__init_config_with_defaults(self): """ Test if configuration defaults point to the correct place. """ @@ -62,23 +66,16 @@ class AuxMethodsTestCase(BaseSoledadTest): def __init__(self): pass - # instantiate without initializing so we just test _init_config() + # instantiate without initializing so we just test + # _init_config_with_defaults() sol = SoledadMock() sol._passphrase = u'' - sol._secrets_path = None - sol._local_db_path = None sol._server_url = '' - sol._init_config() - # assert value of secrets_path - self.assertEquals( - os.path.join( - sol.DEFAULT_PREFIX, Soledad.STORAGE_SECRETS_FILE_NAME), - sol._secrets_path) + sol._init_config_with_defaults() # assert value of local_db_path self.assertEquals( - os.path.join(sol.DEFAULT_PREFIX, 'soledad.u1db'), + os.path.join(sol.default_prefix, 'soledad.u1db'), sol.local_db_path) - sol.close() def test__init_config_from_params(self): """ @@ -93,43 +90,56 @@ class AuxMethodsTestCase(BaseSoledadTest): cert_file=None) self.assertEqual( os.path.join(self.tempdir, 'value_3'), - sol.secrets_path) + sol.secrets.secrets_path) self.assertEqual( os.path.join(self.tempdir, 'value_2'), sol.local_db_path) - self.assertEqual('value_1', sol.server_url) + self.assertEqual('value_1', sol._server_url) sol.close() def test_change_passphrase(self): """ Test if passphrase can be changed. """ + prefix = '_change_passphrase' sol = self._soledad_instance( 'leap@leap.se', passphrase=u'123', - prefix=self.rand_prefix, + prefix=prefix, ) - doc = sol.create_doc({'simple': 'doc'}) - doc_id = doc.doc_id - - # change the passphrase - sol.change_passphrase(u'654321') - sol.close() - self.assertRaises( - WrongMacError, - self._soledad_instance, 'leap@leap.se', - passphrase=u'123', - prefix=self.rand_prefix) - - # use new passphrase and retrieve doc - sol2 = self._soledad_instance( - 'leap@leap.se', - passphrase=u'654321', - prefix=self.rand_prefix) - doc2 = sol2.get_doc(doc_id) - self.assertEqual(doc, doc2) - sol2.close() + def _change_passphrase(doc1): + self._doc1 = doc1 + sol.change_passphrase(u'654321') + sol.close() + + def _assert_wrong_password_raises(results): + self.assertRaises( + WrongMacError, + self._soledad_instance, 'leap@leap.se', + passphrase=u'123', + prefix=prefix) + + def _instantiate_with_new_passphrase(results): + sol2 = self._soledad_instance( + 'leap@leap.se', + passphrase=u'654321', + prefix=prefix) + self._sol2 = sol2 + return sol2.get_doc(self._doc1.doc_id) + + def _assert_docs_are_equal(doc2): + self.assertEqual(self._doc1, doc2) + self._sol2.close() + + d = sol.create_doc({'simple': 'doc'}) + d.addCallback(_change_passphrase) + d.addCallback(_assert_wrong_password_raises) + d.addCallback(_instantiate_with_new_passphrase) + d.addCallback(_assert_docs_are_equal) + d.addCallback(lambda _: sol.close()) + + return d def test_change_passphrase_with_short_passphrase_raises(self): """ @@ -150,7 +160,7 @@ class AuxMethodsTestCase(BaseSoledadTest): Assert passphrase getter works fine. """ sol = self._soledad_instance() - self.assertEqual('123', sol.passphrase) + self.assertEqual('123', sol._passphrase) sol.close() @@ -175,7 +185,7 @@ class SoledadSharedDBTestCase(BaseSoledadTest): doc_id = self._soledad.secrets._shared_db_doc_id() self._soledad.secrets._get_secrets_from_shared_db() self.assertTrue( - self._soledad._shared_db().get_doc.assert_called_with( + self._soledad.shared_db.get_doc.assert_called_with( doc_id) is None, 'Wrong doc_id when fetching recovery document.') @@ -186,11 +196,11 @@ class SoledadSharedDBTestCase(BaseSoledadTest): doc_id = self._soledad.secrets._shared_db_doc_id() self._soledad.secrets._put_secrets_in_shared_db() self.assertTrue( - self._soledad._shared_db().get_doc.assert_called_with( + self._soledad.shared_db.get_doc.assert_called_with( doc_id) is None, 'Wrong doc_id when fetching recovery document.') self.assertTrue( - self._soledad._shared_db.put_doc.assert_called_with( + self._soledad.shared_db.put_doc.assert_called_with( self._doc_put) is None, 'Wrong document when putting recovery document.') self.assertTrue( @@ -285,8 +295,8 @@ class SoledadSignalingTestCase(BaseSoledadTest): ADDRESS, ) # assert db was locked and unlocked - sol._shared_db.lock.assert_called_with() - sol._shared_db.unlock.assert_called_with('atoken') + sol.shared_db.lock.assert_called_with() + sol.shared_db.unlock.assert_called_with('atoken') sol.close() def test_stage2_bootstrap_signals(self): @@ -299,25 +309,15 @@ class SoledadSignalingTestCase(BaseSoledadTest): # create a document with secrets doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id()) doc.content = sol.secrets._export_recovery_document() - - class Stage2MockSharedDB(object): - - get_doc = Mock(return_value=doc) - put_doc = Mock() - lock = Mock(return_value=('atoken', 300)) - unlock = Mock() - - def __call__(self): - return self - sol.close() # reset mock soledad.client.secrets.events.signal.reset_mock() # get a fresh instance so it emits all bootstrap signals + shared_db = self.get_default_shared_mock(get_doc_return_value=doc) sol = self._soledad_instance( secrets_path='alternative_stage2.json', local_db_path='alternative_stage2.u1db', - shared_db_class=Stage2MockSharedDB) + shared_db_class=shared_db) # reverse call order so we can verify in the order the signals were # expected soledad.client.secrets.events.signal.mock_calls.reverse() @@ -355,33 +355,17 @@ class SoledadSignalingTestCase(BaseSoledadTest): sol = self._soledad_instance() # mock the actual db sync so soledad does not try to connect to the # server - sol._db.sync = Mock() - # do the sync - sol.sync() - # assert the signal has been emitted - soledad.client.signal.assert_called_with( - proto.SOLEDAD_DONE_DATA_SYNC, - ADDRESS, - ) - sol.close() - - def test_need_sync_signals(self): - """ - Test Soledad emits SOLEDAD_CREATING_KEYS signal. - """ - soledad.client.signal.reset_mock() - sol = self._soledad_instance() - # mock the sync target - old_get_sync_info = SoledadSyncTarget.get_sync_info - SoledadSyncTarget.get_sync_info = Mock(return_value=[0, 0, 0, 0, 2]) - # mock our generation so soledad thinks there's new data to sync - sol._db._get_generation = Mock(return_value=1) - # check for new data to sync - sol.need_sync('http://provider/userdb') - # assert the signal has been emitted - soledad.client.signal.assert_called_with( - proto.SOLEDAD_NEW_DATA_TO_SYNC, - ADDRESS, - ) - SoledadSyncTarget.get_sync_info = old_get_sync_info - sol.close() + sol._dbsyncer.sync = Mock() + + def _assert_done_data_sync_signal_emitted(results): + # assert the signal has been emitted + soledad.client.signal.assert_called_with( + proto.SOLEDAD_DONE_DATA_SYNC, + ADDRESS, + ) + sol.close() + + # do the sync and assert signal was emitted + d = sol.sync() + d.addCallback(_assert_done_data_sync_signal_emitted) + return d diff --git a/common/src/leap/soledad/common/tests/test_soledad_app.py b/common/src/leap/soledad/common/tests/test_soledad_app.py new file mode 100644 index 00000000..6efae1d6 --- /dev/null +++ b/common/src/leap/soledad/common/tests/test_soledad_app.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# test_soledad_app.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Test ObjectStore and Couch backend bits. +""" + + +from testscenarios import TestWithScenarios + +from leap.soledad.common.tests.util import BaseSoledadTest +from leap.soledad.common.tests.util import make_soledad_document_for_test +from leap.soledad.common.tests.util import make_soledad_app +from leap.soledad.common.tests.util import make_token_soledad_app +from leap.soledad.common.tests.util import make_token_http_database_for_test +from leap.soledad.common.tests.util import copy_token_http_database_for_test +from leap.soledad.common.tests.u1db_tests import test_backends + + +#----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +#----------------------------------------------------------------------------- + +LEAP_SCENARIOS = [ + ('http', { + 'make_database_for_test': test_backends.make_http_database_for_test, + 'copy_database_for_test': test_backends.copy_http_database_for_test, + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_soledad_app}), +] + + +class SoledadTests( + TestWithScenarios, test_backends.AllDatabaseTests, BaseSoledadTest): + + scenarios = LEAP_SCENARIOS + [ + ('token_http', {'make_database_for_test': + make_token_http_database_for_test, + 'copy_database_for_test': + copy_token_http_database_for_test, + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_token_soledad_app, + }) + ] diff --git a/common/src/leap/soledad/common/tests/test_soledad_doc.py b/common/src/leap/soledad/common/tests/test_soledad_doc.py index 0952de6d..4a67f80a 100644 --- a/common/src/leap/soledad/common/tests/test_soledad_doc.py +++ b/common/src/leap/soledad/common/tests/test_soledad_doc.py @@ -17,28 +17,30 @@ """ Test Leap backend bits: soledad docs """ -from leap.soledad.common.tests import BaseSoledadTest +from testscenarios import TestWithScenarios + from leap.soledad.common.tests.u1db_tests import test_document -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests import test_sync_target as st +from leap.soledad.common.tests.util import BaseSoledadTest +from leap.soledad.common.tests.util import make_soledad_document_for_test + #----------------------------------------------------------------------------- # The following tests come from `u1db.tests.test_document`. #----------------------------------------------------------------------------- - -class TestSoledadDocument(test_document.TestDocument, BaseSoledadTest): +class TestSoledadDocument( + TestWithScenarios, + test_document.TestDocument, BaseSoledadTest): scenarios = ([( 'leap', { - 'make_document_for_test': st.make_leap_document_for_test})]) + 'make_document_for_test': make_soledad_document_for_test})]) -class TestSoledadPyDocument(test_document.TestPyDocument, BaseSoledadTest): +class TestSoledadPyDocument( + TestWithScenarios, + test_document.TestPyDocument, BaseSoledadTest): scenarios = ([( 'leap', { - 'make_document_for_test': st.make_leap_document_for_test})]) - - -load_tests = tests.load_with_scenarios + 'make_document_for_test': make_soledad_document_for_test})]) diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py index 78e2f01b..ceb095b8 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py @@ -20,10 +20,11 @@ Test sqlcipher backend internals. import os import time import threading - +import tempfile +import shutil from pysqlcipher import dbapi2 - +from testscenarios import TestWithScenarios # u1db stuff. from u1db import ( @@ -34,17 +35,16 @@ from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase # soledad stuff. +from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument from leap.soledad.client.sqlcipher import ( SQLCipherDatabase, SQLCipherOptions, DatabaseIsNotEncrypted, - initialize_sqlcipher_db, ) # u1db tests stuff. -from leap.common.testing.basetest import BaseLeapTest from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_sqlite_backend from leap.soledad.common.tests.u1db_tests import test_backends @@ -53,12 +53,12 @@ from leap.soledad.common.tests.util import ( make_sqlcipher_database_for_test, copy_sqlcipher_database_for_test, PASSWORD, + BaseSoledadTest, ) def sqlcipher_open(path, passphrase, create=True, document_factory=None): return SQLCipherDatabase( - None, SQLCipherOptions(path, passphrase, create=create)) @@ -93,30 +93,34 @@ SQLCIPHER_SCENARIOS = [ ] -class SQLCipherTests(test_backends.AllDatabaseTests): +class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): scenarios = SQLCIPHER_SCENARIOS -class SQLCipherDatabaseTests(test_backends.LocalDatabaseTests): +class SQLCipherDatabaseTests(TestWithScenarios, test_backends.LocalDatabaseTests): scenarios = SQLCIPHER_SCENARIOS class SQLCipherValidateGenNTransIdTests( + TestWithScenarios, test_backends.LocalDatabaseValidateGenNTransIdTests): scenarios = SQLCIPHER_SCENARIOS class SQLCipherValidateSourceGenTests( + TestWithScenarios, test_backends.LocalDatabaseValidateSourceGenTests): scenarios = SQLCIPHER_SCENARIOS class SQLCipherWithConflictsTests( + TestWithScenarios, test_backends.LocalDatabaseWithConflictsTests): scenarios = SQLCIPHER_SCENARIOS -class SQLCipherIndexTests(test_backends.DatabaseIndexTests): +class SQLCipherIndexTests( + TestWithScenarios, test_backends.DatabaseIndexTests): scenarios = SQLCIPHER_SCENARIOS @@ -124,7 +128,7 @@ class SQLCipherIndexTests(test_backends.DatabaseIndexTests): # The following tests come from `u1db.tests.test_sqlite_backend`. #----------------------------------------------------------------------------- -class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): +class TestSQLCipherDatabase(TestWithScenarios, test_sqlite_backend.TestSQLiteDatabase): def test_atomic_initialize(self): # This test was modified to ensure that db2.close() is called within @@ -137,11 +141,11 @@ class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): class SQLCipherDatabaseTesting(SQLCipherDatabase): _index_storage_value = "testing" - def __init__(self, soledad_crypto, dbname, ntry): + def __init__(self, dbname, ntry): self._try = ntry self._is_initialized_invocations = 0 SQLCipherDatabase.__init__( - self, soledad_crypto, + self, SQLCipherOptions(dbname, PASSWORD)) def _is_initialized(self, c): @@ -161,15 +165,14 @@ class TestSQLCipherDatabase(test_sqlite_backend.TestSQLiteDatabase): def run(self): try: - db2 = SQLCipherDatabaseTesting(None, dbname, 2) + db2 = SQLCipherDatabaseTesting(dbname, 2) except Exception, e: SecondTry.outcome2.append(e) else: SecondTry.outcome2.append(db2) - self.close() t2 = SecondTry() - db1 = SQLCipherDatabaseTesting(None, dbname, 1) + db1 = SQLCipherDatabaseTesting(dbname, 1) t2.join() self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) @@ -368,7 +371,7 @@ class SQLCipherOpen(test_open.TestU1DBOpen): # Tests for actual encryption of the database #----------------------------------------------------------------------------- -class SQLCipherEncryptionTest(BaseLeapTest): +class SQLCipherEncryptionTest(BaseSoledadTest): """ Tests to guarantee SQLCipher is indeed encrypting data when storing. """ @@ -379,11 +382,37 @@ class SQLCipherEncryptionTest(BaseLeapTest): os.unlink(dbfile) def setUp(self): + # the following come from BaseLeapTest.setUpClass, because + # twisted.trial doesn't support such class methods for setting up + # test classes. + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + # this is our own stuff self.DB_FILE = os.path.join(self.tempdir, 'test.db') self._delete_dbfiles() def tearDown(self): self._delete_dbfiles() + # the following come from BaseLeapTest.tearDownClass, because + # twisted.trial doesn't support such class methods for tearing down + # test classes. + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check! please do not wipe my home... + # XXX needs to adapt to non-linuces + soledad_assert( + self.tempdir.startswith('/tmp/leap_tests-') or + self.tempdir.startswith('/var/folder'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(self.tempdir) def test_try_to_open_encrypted_db_with_sqlite_backend(self): """ @@ -426,6 +455,3 @@ class SQLCipherEncryptionTest(BaseLeapTest): "dbs.") except DatabaseIsNotEncrypted: pass - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py index ad2a06b3..83c3449e 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py @@ -19,12 +19,14 @@ Test sqlcipher backend sync. """ +import os import simplejson as json from u1db import ( sync, vectorclock, ) +from testscenarios import TestWithScenarios from leap.soledad.common.crypto import ENC_SCHEME_KEY from leap.soledad.client.target import SoledadSyncTarget @@ -33,15 +35,12 @@ from leap.soledad.client.sqlcipher import ( SQLCipherDatabase, ) - -from leap.soledad.common.tests import u1db_tests as tests, BaseSoledadTest +from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_sync -from leap.soledad.common.tests.test_sqlcipher import ( - SQLCIPHER_SCENARIOS, - make_document_for_test, -) +from leap.soledad.common.tests.test_sqlcipher import SQLCIPHER_SCENARIOS from leap.soledad.common.tests.util import ( make_soledad_app, + BaseSoledadTest, SoledadWithCouchServerMixin, ) @@ -50,15 +49,7 @@ from leap.soledad.common.tests.util import ( # The following tests come from `u1db.tests.test_sync`. #----------------------------------------------------------------------------- -sync_scenarios = [] -for name, scenario in SQLCIPHER_SCENARIOS: - scenario = dict(scenario) - scenario['do_sync'] = test_sync.sync_via_synchronizer - sync_scenarios.append((name, scenario)) - scenario = dict(scenario) - - -def sync_via_synchronizer_and_leap(test, db_source, db_target, +def sync_via_synchronizer_and_soledad(test, db_source, db_target, trace_hook=None, trace_hook_shallow=None): if trace_hook: test.skipTest("full trace hook unsupported over http") @@ -71,17 +62,16 @@ def sync_via_synchronizer_and_leap(test, db_source, db_target, return sync.Synchronizer(db_source, target).sync() -sync_scenarios.append(('pyleap', { - 'make_database_for_test': test_sync.make_database_for_http_test, - 'copy_database_for_test': test_sync.copy_database_for_http_test, - 'make_document_for_test': make_document_for_test, - 'make_app_with_state': tests.test_remote_sync_target.make_http_app, - 'do_sync': test_sync.sync_via_synchronizer, -})) +sync_scenarios = [] +for name, scenario in SQLCIPHER_SCENARIOS: + scenario['do_sync'] = test_sync.sync_via_synchronizer + sync_scenarios.append((name, scenario)) class SQLCipherDatabaseSyncTests( - test_sync.DatabaseSyncTests, BaseSoledadTest): + TestWithScenarios, + test_sync.DatabaseSyncTests, + BaseSoledadTest): """ Test for succesfull sync between SQLCipher and LeapBackend. @@ -92,8 +82,8 @@ class SQLCipherDatabaseSyncTests( scenarios = sync_scenarios - def setUp(self): - test_sync.DatabaseSyncTests.setUp(self) + #def setUp(self): + # test_sync.DatabaseSyncTests.setUp(self) def tearDown(self): test_sync.DatabaseSyncTests.tearDown(self) @@ -112,8 +102,6 @@ class SQLCipherDatabaseSyncTests( and isinstance(self.db3, SQLCipherDatabase): self.db3.close() - - def test_sync_autoresolves(self): """ Test for sync autoresolve remote. @@ -321,12 +309,14 @@ target_scenarios = [ 'create_db_and_target': _make_local_db_and_token_http_target, # 'make_app_with_state': tests.test_remote_sync_target.make_http_app, 'make_app_with_state': make_soledad_app, - 'do_sync': test_sync.sync_via_synchronizer}), + 'do_sync': sync_via_synchronizer_and_soledad}), ] class SQLCipherSyncTargetTests( - SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): + TestWithScenarios, + SoledadWithCouchServerMixin, + test_sync.DatabaseSyncTargetTests): scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, target_scenarios)) @@ -406,4 +396,3 @@ class SQLCipherSyncTargetTests( self.db._last_exchange_log['return'], {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py index 0433fac9..893df56b 100644 --- a/common/src/leap/soledad/common/tests/test_sync.py +++ b/common/src/leap/soledad/common/tests/test_sync.py @@ -16,43 +16,35 @@ # along with this program. If not, see . -import mock -import os import json import tempfile import threading import time + from urlparse import urljoin +from twisted.internet import defer + +from testscenarios import TestWithScenarios from leap.soledad.common import couch +from leap.soledad.client import target +from leap.soledad.client import sync +from leap.soledad.server import SoledadApp -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests import test_sync_target from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import ( - TestCaseWithServer, - simple_doc, - test_backends, - test_sync -) -from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.test_target_soledad import ( - make_token_soledad_app, - make_leap_document_for_test, -) -from leap.soledad.common.tests.test_sync_target import token_leap_sync_target -from leap.soledad.client import ( - Soledad, - target, -) +from leap.soledad.common.tests.u1db_tests import TestCaseWithServer +from leap.soledad.common.tests.u1db_tests import simple_doc +from leap.soledad.common.tests.u1db_tests import test_sync +from leap.soledad.common.tests.util import make_token_soledad_app +from leap.soledad.common.tests.util import make_soledad_document_for_test +from leap.soledad.common.tests.util import token_soledad_sync_target +from leap.soledad.common.tests.util import BaseSoledadTest from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.client.sync import SoledadSynchronizer -from leap.soledad.server import SoledadApp - +from leap.soledad.common.tests.test_couch import CouchDBTestCase class InterruptableSyncTestCase( - CouchDBTestCase, TestCaseWithServer): + BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): """ Tests for encrypted sync using Soledad server backed by a couch database. """ @@ -61,47 +53,9 @@ class InterruptableSyncTestCase( def make_app_with_state(state): return make_token_soledad_app(state) - make_document_for_test = make_leap_document_for_test - - sync_target = token_leap_sync_target + make_document_for_test = make_soledad_document_for_test - def _soledad_instance(self, user='user-uuid', passphrase=u'123', - prefix='', - secrets_path=Soledad.STORAGE_SECRETS_FILE_NAME, - local_db_path='soledad.u1db', server_url='', - cert_file=None, auth_token=None, secret_id=None): - """ - Instantiate Soledad. - """ - - # this callback ensures we save a document which is sent to the shared - # db. - def _put_doc_side_effect(doc): - self._doc_put = doc - - # we need a mocked shared db or else Soledad will try to access the - # network to find if there are uploaded secrets. - class MockSharedDB(object): - - get_doc = mock.Mock(return_value=None) - put_doc = mock.Mock(side_effect=_put_doc_side_effect) - lock = mock.Mock(return_value=('atoken', 300)) - unlock = mock.Mock() - - def __call__(self): - return self - - Soledad._shared_db = MockSharedDB() - return Soledad( - user, - passphrase, - secrets_path=os.path.join(self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, - cert_file=cert_file, - auth_token=auth_token, - secret_id=secret_id) + sync_target = token_soledad_sync_target def make_app(self): self.request_state = couch.CouchServerState( @@ -135,7 +89,8 @@ class InterruptableSyncTestCase( def run(self): while db._get_generation() < 2: - time.sleep(1) + #print "WAITING %d" % db._get_generation() + time.sleep(0.1) self._soledad.stop_sync() time.sleep(1) @@ -143,16 +98,7 @@ class InterruptableSyncTestCase( self.startServer() # instantiate soledad and create a document - sol = self._soledad_instance( - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token' - ) - _, doclist = sol.get_all_docs() - self.assertEqual([], doclist) - - # create many small files - for i in range(0, number_of_docs): - sol.create_doc(json.loads(simple_doc)) + sol = self._soledad_instance(user='user-uuid', server_url=self.getURL()) # ensure remote db exists before syncing db = couch.CouchDatabase.open_database( @@ -164,21 +110,35 @@ class InterruptableSyncTestCase( t = _SyncInterruptor(sol, db) t.start() - # sync with server - sol._server_url = self.getURL() - sol.sync() # this will be interrupted when couch db gen >= 2 - t.join() + d = sol.get_all_docs() + d.addCallback(lambda results: self.assertEqual([], results[1])) - # recover the sync process - sol.sync() + def _create_docs(results): + # create many small files + deferreds = [] + for i in range(0, number_of_docs): + deferreds.append(sol.create_doc(json.loads(simple_doc))) + return defer.DeferredList(deferreds) - gen, doclist = db.get_all_docs() - self.assertEqual(number_of_docs, len(doclist)) - - # delete remote database - db.delete_database() - db.close() - sol.close() + # sync with server + d.addCallback(_create_docs) + d.addCallback(lambda _: sol.get_all_docs()) + d.addCallback(lambda results: self.assertEqual(number_of_docs, len(results[1]))) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: t.join()) + d.addCallback(lambda _: db.get_all_docs()) + d.addCallback(lambda results: self.assertNotEqual(number_of_docs, len(results[1]))) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: db.get_all_docs()) + d.addCallback(lambda results: self.assertEqual(number_of_docs, len(results[1]))) + + def _tear_down(results): + db.delete_database() + db.close() + sol.close() + + d.addCallback(_tear_down) + return d def make_soledad_app(state): @@ -186,6 +146,7 @@ def make_soledad_app(state): class TestSoledadDbSync( + TestWithScenarios, SoledadWithCouchServerMixin, test_sync.TestDbSync): """ @@ -198,7 +159,7 @@ class TestSoledadDbSync( 'make_database_for_test': tests.make_memory_database_for_test, }), ('py-token-http', { - 'make_app_with_state': test_sync_target.make_token_soledad_app, + 'make_app_with_state': make_token_soledad_app, 'make_database_for_test': tests.make_memory_database_for_test, 'token': True }), @@ -211,10 +172,11 @@ class TestSoledadDbSync( """ Need to explicitely invoke inicialization on all bases. """ - tests.TestCaseWithServer.setUp(self) - self.main_test_class = test_sync.TestDbSync + #tests.TestCaseWithServer.setUp(self) + #self.main_test_class = test_sync.TestDbSync SoledadWithCouchServerMixin.setUp(self) self.startServer() + self.db = self.make_database_for_test(self, 'test1') self.db2 = couch.CouchDatabase.open_database( urljoin( 'http://localhost:' + str(self.wrapper.port), 'test'), @@ -227,7 +189,7 @@ class TestSoledadDbSync( """ self.db2.delete_database() SoledadWithCouchServerMixin.tearDown(self) - tests.TestCaseWithServer.tearDown(self) + #tests.TestCaseWithServer.tearDown(self) def do_sync(self, target_name): """ @@ -240,7 +202,7 @@ class TestSoledadDbSync( 'token': 'auth-token', }}) target_url = self.getURL(target_name) - return SoledadSynchronizer( + return sync.SoledadSynchronizer( self.db, target.SoledadSyncTarget( target_url, @@ -254,8 +216,10 @@ class TestSoledadDbSync( Adapted to check for encrypted content. """ + doc1 = self.db.create_doc_from_json(tests.simple_doc) doc2 = self.db2.create_doc_from_json(tests.nested_doc) + local_gen_before_sync = self.do_sync('test') gen, _, changes = self.db.whats_changed(local_gen_before_sync) self.assertEqual(1, len(changes)) @@ -287,6 +251,3 @@ class TestSoledadDbSync( s_gen, _ = db3._get_replica_gen_and_trans_id('test1') self.assertEqual(1, t_gen) self.assertEqual(1, s_gen) - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_sync_deferred.py b/common/src/leap/soledad/common/tests/test_sync_deferred.py index 07a9742b..26889aff 100644 --- a/common/src/leap/soledad/common/tests/test_sync_deferred.py +++ b/common/src/leap/soledad/common/tests/test_sync_deferred.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # test_sync_deferred.py # Copyright (C) 2014 LEAP # @@ -21,28 +20,31 @@ import time import os import random import string +import shutil + from urlparse import urljoin -from leap.soledad.common.tests import u1db_tests as tests, ADDRESS +from leap.soledad.common import couch +from leap.soledad.client.sqlcipher import ( + SQLCipherOptions, + SQLCipherDatabase, + SQLCipherU1DBSync, +) + +from testscenarios import TestWithScenarios + +from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import test_sync +from leap.soledad.common.tests.util import ADDRESS +from leap.soledad.common.tests.util import SoledadWithCouchServerMixin +from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common import couch -from leap.soledad.client import target -from leap.soledad.client.sync import SoledadSynchronizer # Just to make clear how this test is different... :) DEFER_DECRYPTION = True WAIT_STEP = 1 MAX_WAIT = 10 - - -from leap.soledad.client.sqlcipher import open as open_sqlcipher -from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.common.tests.util import make_soledad_app - - DBPASS = "pass" @@ -54,8 +56,10 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): defer_sync_encryption = True def setUp(self): + SoledadWithCouchServerMixin.setUp(self) # config info self.db1_file = os.path.join(self.tempdir, "db1.u1db") + os.unlink(self.db1_file) self.db_pass = DBPASS self.email = ADDRESS @@ -64,17 +68,21 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): # each local db. self.rand_prefix = ''.join( map(lambda x: random.choice(string.ascii_letters), range(6))) - # initialize soledad by hand so we can control keys - self._soledad = self._soledad_instance( - prefix=self.rand_prefix, user=self.email) - - # open test dbs: db1 will be the local sqlcipher db - # (which instantiates a syncdb) - self.db1 = open_sqlcipher(self.db1_file, DBPASS, create=True, - document_factory=SoledadDocument, - crypto=self._soledad._crypto, - defer_encryption=True, - sync_db_key=DBPASS) + + # open test dbs: db1 will be the local sqlcipher db (which + # instantiates a syncdb). We use the self._soledad instance that was + # already created on some setUp method. + import binascii + tohex = binascii.b2a_hex + key = tohex(self._soledad.secrets.get_local_storage_key()) + sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) + dbpath = self._soledad._local_db_path + + self.opts = SQLCipherOptions( + dbpath, key, is_raw_key=True, create=False, + defer_encryption=True, sync_db_key=sync_db_key) + self.db1 = SQLCipherDatabase(self.opts) + self.db2 = couch.CouchDatabase.open_database( urljoin( 'http://localhost:' + str(self.wrapper.port), 'test'), @@ -87,20 +95,8 @@ class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): self._soledad.close() # XXX should not access "private" attrs - import shutil shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) - - -#SQLCIPHER_SCENARIOS = [ -# ('http', { -# #'make_app_with_state': test_sync_target.make_token_soledad_app, -# 'make_app_with_state': make_soledad_app, -# 'make_database_for_test': ts.make_sqlcipher_database_for_test, -# 'copy_database_for_test': ts.copy_sqlcipher_database_for_test, -# 'make_document_for_test': ts.make_document_for_test, -# 'token': True -# }), -#] + SoledadWithCouchServerMixin.tearDown(self) class SyncTimeoutError(Exception): @@ -111,8 +107,9 @@ class SyncTimeoutError(Exception): class TestSoledadDbSyncDeferredEncDecr( - BaseSoledadDeferredEncTest, - test_sync.TestDbSync): + TestWithScenarios, + test_sync.TestDbSync, + BaseSoledadDeferredEncTest): """ Test db.sync remote sync shortcut. Case with deferred encryption and decryption: using the intermediate @@ -129,13 +126,17 @@ class TestSoledadDbSyncDeferredEncDecr( oauth = False token = True + def make_app(self): + self.request_state = couch.CouchServerState( + self._couch_url, 'shared', 'tokens') + return self.make_app_with_state(self.request_state) + def setUp(self): """ Need to explicitely invoke inicialization on all bases. """ - tests.TestCaseWithServer.setUp(self) - self.main_test_class = test_sync.TestDbSync BaseSoledadDeferredEncTest.setUp(self) + self.server = self.server_thread = None self.startServer() self.syncer = None @@ -143,8 +144,10 @@ class TestSoledadDbSyncDeferredEncDecr( """ Need to explicitely invoke destruction on all bases. """ + dbsyncer = getattr(self, 'dbsyncer', None) + if dbsyncer: + dbsyncer.close() BaseSoledadDeferredEncTest.tearDown(self) - tests.TestCaseWithServer.tearDown(self) def do_sync(self, target_name): """ @@ -152,25 +155,20 @@ class TestSoledadDbSyncDeferredEncDecr( and Token auth. """ if self.token: - extra = dict(creds={'token': { + creds={'token': { 'uuid': 'user-uuid', 'token': 'auth-token', - }}) + }} target_url = self.getURL(target_name) - syncdb = getattr(self.db1, "_sync_db", None) - - syncer = SoledadSynchronizer( - self.db1, - target.SoledadSyncTarget( - target_url, - crypto=self._soledad._crypto, - sync_db=syncdb, - **extra)) - # Keep a reference to be able to know when the sync - # has finished. - self.syncer = syncer - return syncer.sync( - autocreate=True, defer_decryption=DEFER_DECRYPTION) + + # get a u1db syncer + crypto = self._soledad._crypto + replica_uid = self.db1._replica_uid + dbsyncer = SQLCipherU1DBSync(self.opts, crypto, replica_uid, + defer_encryption=True) + self.dbsyncer = dbsyncer + return dbsyncer.sync(target_url, creds=creds, + autocreate=True,defer_decryption=DEFER_DECRYPTION) else: return test_sync.TestDbSync.do_sync(self, target_name) @@ -195,28 +193,30 @@ class TestSoledadDbSyncDeferredEncDecr( """ doc1 = self.db1.create_doc_from_json(tests.simple_doc) doc2 = self.db2.create_doc_from_json(tests.nested_doc) + d = self.do_sync('test') - import time - # need to give time to the encryption to proceed - # TODO should implement a defer list to subscribe to the all-decrypted - # event - time.sleep(2) + def _assert_successful_sync(results): + import time + # need to give time to the encryption to proceed + # TODO should implement a defer list to subscribe to the all-decrypted + # event + time.sleep(2) + local_gen_before_sync = results + self.wait_for_sync() - local_gen_before_sync = self.do_sync('test') - self.wait_for_sync() + gen, _, changes = self.db1.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) - gen, _, changes = self.db1.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) + self.assertGetEncryptedDoc( + self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) + d.addCallback(_assert_successful_sync) + return d def test_db_sync_autocreate(self): pass - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py index 45009f4e..3792159a 100644 --- a/common/src/leap/soledad/common/tests/test_sync_target.py +++ b/common/src/leap/soledad/common/tests/test_sync_target.py @@ -19,82 +19,38 @@ Test Leap backend bits: sync target """ import cStringIO import os - +import time import simplejson as json import u1db +import random +import string +import shutil + +from testscenarios import TestWithScenarios +from urlparse import urljoin -from u1db.remote import http_database +from leap.soledad.client import target +from leap.soledad.client import crypto +from leap.soledad.client.sqlcipher import SQLCipherU1DBSync +from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client import ( - target, - auth, - crypto, - sync, -) +from leap.soledad.common import couch from leap.soledad.common.document import SoledadDocument from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests.util import ( - make_sqlcipher_database_for_test, - make_soledad_app, - make_token_soledad_app, - SoledadWithCouchServerMixin, -) -from leap.soledad.common.tests.u1db_tests import test_backends +from leap.soledad.common.tests.util import make_sqlcipher_database_for_test +from leap.soledad.common.tests.util import make_soledad_app +from leap.soledad.common.tests.util import make_token_soledad_app +from leap.soledad.common.tests.util import make_soledad_document_for_test +from leap.soledad.common.tests.util import token_soledad_sync_target +from leap.soledad.common.tests.util import BaseSoledadTest +from leap.soledad.common.tests.util import SoledadWithCouchServerMixin +from leap.soledad.common.tests.util import ADDRESS from leap.soledad.common.tests.u1db_tests import test_remote_sync_target from leap.soledad.common.tests.u1db_tests import test_sync -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -#----------------------------------------------------------------------------- - -def make_leap_document_for_test(test, doc_id, rev, content, - has_conflicts=False): - return SoledadDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -def make_token_http_database_for_test(test, replica_uid): - test.startServer() - test.request_state._create_database(replica_uid) - - class _HTTPDatabaseWithToken( - http_database.HTTPDatabase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - http_db = _HTTPDatabaseWithToken(test.getURL('test')) - http_db.set_token_credentials('user-uuid', 'auth-token') - return http_db - - -def copy_token_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') - return http_db - - #----------------------------------------------------------------------------- # The following tests come from `u1db.tests.test_remote_sync_target`. #----------------------------------------------------------------------------- @@ -122,12 +78,6 @@ class TestSoledadParsingSyncStream( target. """ - def setUp(self): - test_remote_sync_target.TestParsingSyncStream.setUp(self) - - def tearDown(self): - test_remote_sync_target.TestParsingSyncStream.tearDown(self) - def test_extra_comma(self): """ Test adapted to use encrypted content. @@ -209,17 +159,6 @@ class TestSoledadParsingSyncStream( # functions for TestRemoteSyncTargets # -def leap_sync_target(test, path): - return target.SoledadSyncTarget( - test.getURL(path), crypto=test._soledad._crypto) - - -def token_leap_sync_target(test, path): - st = leap_sync_target(test, path) - st.set_token_credentials('user-uuid', 'auth-token') - return st - - def make_local_db_and_soledad_target(test, path='test'): test.startServer() db = test.request_state._create_database(os.path.basename(path)) @@ -235,32 +174,32 @@ def make_local_db_and_token_soledad_target(test): class TestSoledadSyncTarget( + TestWithScenarios, SoledadWithCouchServerMixin, test_remote_sync_target.TestRemoteSyncTargets): scenarios = [ ('token_soledad', {'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_leap_document_for_test, + 'make_document_for_test': make_soledad_document_for_test, 'create_db_and_target': make_local_db_and_token_soledad_target, 'make_database_for_test': make_sqlcipher_database_for_test, - 'sync_target': token_leap_sync_target}), + 'sync_target': token_soledad_sync_target}), ] def setUp(self): - tests.TestCaseWithServer.setUp(self) - self.main_test_class = test_remote_sync_target.TestRemoteSyncTargets + TestWithScenarios.setUp(self) SoledadWithCouchServerMixin.setUp(self) self.startServer() self.db1 = make_sqlcipher_database_for_test(self, 'test1') self.db2 = self.request_state._create_database('test2') def tearDown(self): - SoledadWithCouchServerMixin.tearDown(self) - tests.TestCaseWithServer.tearDown(self) - db2, _ = self.request_state.ensure_database('test2') - db2.delete_database() + #db2, _ = self.request_state.ensure_database('test2') + self.db2.delete_database() self.db1.close() + SoledadWithCouchServerMixin.tearDown(self) + TestWithScenarios.tearDown(self) def test_sync_exchange_send(self): """ @@ -268,7 +207,6 @@ class TestSoledadSyncTarget( This test was adapted to decrypt remote content before assert. """ - self.startServer() db = self.request_state._create_database('test') remote_target = self.getSyncTarget('test') other_docs = [] @@ -289,14 +227,9 @@ class TestSoledadSyncTarget( """ Test for sync exchange failure and retry. - This test was adapted to: - - decrypt remote content before assert. - - not expect a bounced document because soledad has stateful - recoverable sync. + This test was adapted to decrypt remote content before assert. """ - self.startServer() - def blackhole_getstderr(inst): return cStringIO.StringIO() @@ -332,8 +265,9 @@ class TestSoledadSyncTarget( doc2 = self.make_document('doc-here2', 'replica:1', '{"value": "here2"}') - # we do not expect an HTTPError because soledad sync fails gracefully - remote_target.sync_exchange( + self.assertRaises( + u1db.errors.HTTPError, + remote_target.sync_exchange, [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], 'replica', last_known_generation=0, last_known_trans_id=None, return_doc_cb=receive_doc) @@ -364,7 +298,6 @@ class TestSoledadSyncTarget( This test was adapted to decrypt remote content before assert. """ - self.startServer() remote_target = self.getSyncTarget('test') other_docs = [] replica_uid_box = [] @@ -405,7 +338,9 @@ target_scenarios = [ class SoledadDatabaseSyncTargetTests( - SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): + TestWithScenarios, + SoledadWithCouchServerMixin, + test_sync.DatabaseSyncTargetTests): scenarios = ( tests.multiply_scenarios( @@ -500,8 +435,25 @@ class SoledadDatabaseSyncTargetTests( [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) +# Just to make clear how this test is different... :) +DEFER_DECRYPTION = False + +WAIT_STEP = 1 +MAX_WAIT = 10 +DBPASS = "pass" + + +class SyncTimeoutError(Exception): + """ + Dummy exception to notify timeout during sync. + """ + pass + + class TestSoledadDbSync( - SoledadWithCouchServerMixin, test_sync.TestDbSync): + TestWithScenarios, + SoledadWithCouchServerMixin, + test_sync.TestDbSync): """Test db.sync remote sync shortcut""" scenarios = [ @@ -516,13 +468,67 @@ class TestSoledadDbSync( oauth = False token = False + + def make_app(self): + self.request_state = couch.CouchServerState( + self._couch_url, 'shared', 'tokens') + return self.make_app_with_state(self.request_state) + def setUp(self): - self.main_test_class = test_sync.TestDbSync + """ + Need to explicitely invoke inicialization on all bases. + """ SoledadWithCouchServerMixin.setUp(self) + self.server = self.server_thread = None + self.startServer() + self.syncer = None + + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + os.unlink(self.db1_file) + self.db_pass = DBPASS + self.email = ADDRESS + + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + + # open test dbs: db1 will be the local sqlcipher db (which + # instantiates a syncdb). We use the self._soledad instance that was + # already created on some setUp method. + import binascii + tohex = binascii.b2a_hex + key = tohex(self._soledad.secrets.get_local_storage_key()) + sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) + dbpath = self._soledad._local_db_path + + self.opts = SQLCipherOptions( + dbpath, key, is_raw_key=True, create=False, + defer_encryption=True, sync_db_key=sync_db_key) + self.db1 = SQLCipherDatabase(self.opts) + + self.db2 = couch.CouchDatabase.open_database( + urljoin( + 'http://localhost:' + str(self.wrapper.port), 'test'), + create=True, + ensure_ddocs=True) def tearDown(self): + """ + Need to explicitely invoke destruction on all bases. + """ + dbsyncer = getattr(self, 'dbsyncer', None) + if dbsyncer: + dbsyncer.close() + self.db1.close() + self.db2.close() + self._soledad.close() + + # XXX should not access "private" attrs + shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) SoledadWithCouchServerMixin.tearDown(self) - self.db.close() def do_sync(self, target_name): """ @@ -530,44 +536,71 @@ class TestSoledadDbSync( and Token auth. """ if self.token: - extra = dict(creds={'token': { + creds={'token': { 'uuid': 'user-uuid', 'token': 'auth-token', - }}) + }} target_url = self.getURL(target_name) - return sync.SoledadSynchronizer( - self.db, - target.SoledadSyncTarget( - target_url, - crypto=self._soledad._crypto, - **extra)).sync(autocreate=True, - defer_decryption=False) + + # get a u1db syncer + crypto = self._soledad._crypto + replica_uid = self.db1._replica_uid + dbsyncer = SQLCipherU1DBSync(self.opts, crypto, replica_uid, + defer_encryption=True) + self.dbsyncer = dbsyncer + return dbsyncer.sync(target_url, creds=creds, + autocreate=True,defer_decryption=DEFER_DECRYPTION) else: return test_sync.TestDbSync.do_sync(self, target_name) + def wait_for_sync(self): + """ + Wait for sync to finish. + """ + wait = 0 + syncer = self.syncer + if syncer is not None: + while syncer.syncing: + time.sleep(WAIT_STEP) + wait += WAIT_STEP + if wait >= MAX_WAIT: + raise SyncTimeoutError + def test_db_sync(self): """ Test sync. Adapted to check for encrypted content. """ - doc1 = self.db.create_doc_from_json(tests.simple_doc) + doc1 = self.db1.create_doc_from_json(tests.simple_doc) doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = self.do_sync('test2') - gen, _, changes = self.db.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) + d = self.do_sync('test') + + def _assert_successful_sync(results): + import time + # need to give time to the encryption to proceed + # TODO should implement a defer list to subscribe to the all-decrypted + # event + time.sleep(2) + local_gen_before_sync = results + self.wait_for_sync() + + gen, _, changes = self.db1.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) + + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) + + self.assertGetEncryptedDoc( + self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) + + d.addCallback(_assert_successful_sync) + return d def test_db_sync_autocreate(self): """ We bypass this test because we never need to autocreate databases. """ pass - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_target.py b/common/src/leap/soledad/common/tests/test_target.py deleted file mode 100644 index eb5e2874..00000000 --- a/common/src/leap/soledad/common/tests/test_target.py +++ /dev/null @@ -1,797 +0,0 @@ -# -*- coding: utf-8 -*- -# test_target.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Test Leap backend bits. -""" - -import u1db -import os -import simplejson as json -import cStringIO - - -from u1db.sync import Synchronizer -from u1db.remote import ( - http_client, - http_database, -) - -from leap.soledad import client -from leap.soledad.client import ( - target, - auth, - VerifiedHTTPSConnection, -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests.util import ( - make_sqlcipher_database_for_test, - make_soledad_app, - make_token_soledad_app, - SoledadWithCouchServerMixin, -) -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import test_http_database -from leap.soledad.common.tests.u1db_tests import test_http_client -from leap.soledad.common.tests.u1db_tests import test_document -from leap.soledad.common.tests.u1db_tests import test_remote_sync_target -from leap.soledad.common.tests.u1db_tests import test_https -from leap.soledad.common.tests.u1db_tests import test_sync - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -#----------------------------------------------------------------------------- - -def make_leap_document_for_test(test, doc_id, rev, content, - has_conflicts=False): - return SoledadDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -def make_token_http_database_for_test(test, replica_uid): - test.startServer() - test.request_state._create_database(replica_uid) - - class _HTTPDatabaseWithToken( - http_database.HTTPDatabase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - http_db = _HTTPDatabaseWithToken(test.getURL('test')) - http_db.set_token_credentials('user-uuid', 'auth-token') - return http_db - - -def copy_token_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') - return http_db - - -class SoledadTests(test_backends.AllDatabaseTests, BaseSoledadTest): - - scenarios = LEAP_SCENARIOS + [ - ('token_http', {'make_database_for_test': - make_token_http_database_for_test, - 'copy_database_for_test': - copy_token_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_token_soledad_app, - }) - ] - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_client`. -#----------------------------------------------------------------------------- - -class TestSoledadClientBase(test_http_client.TestHTTPClientBase): - """ - This class should be used to test Token auth. - """ - - def getClientWithToken(self, **kwds): - self.startServer() - - class _HTTPClientWithToken( - http_client.HTTPClientBase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - return _HTTPClientWithToken(self.getURL('dbase'), **kwds) - - def test_oauth(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def test_oauth_ctr_creds(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def test_oauth_Unauthorized(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def app(self, environ, start_response): - res = test_http_client.TestHTTPClientBase.app( - self, environ, start_response) - if res is not None: - return res - # mime solead application here. - if '/token' in environ['PATH_INFO']: - auth = environ.get(SoledadTokenAuthMiddleware.HTTP_AUTH_KEY) - if not auth: - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - scheme, encoded = auth.split(None, 1) - if scheme.lower() != 'token': - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - uuid, token = encoded.decode('base64').split(':', 1) - if uuid != 'user-uuid' and token != 'auth-token': - return unauth_err("Incorrect address or token.") - start_response("200 OK", [('Content-Type', 'application/json')]) - return [json.dumps([environ['PATH_INFO'], uuid, token])] - - def test_token(self): - """ - Test if token is sent correctly. - """ - cli = self.getClientWithToken() - cli.set_token_credentials('user-uuid', 'auth-token') - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - - def test_token_ctr_creds(self): - cli = self.getClientWithToken(creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_document`. -#----------------------------------------------------------------------------- - -class TestSoledadDocument(test_document.TestDocument, BaseSoledadTest): - - scenarios = ([( - 'leap', {'make_document_for_test': make_leap_document_for_test})]) - - -class TestSoledadPyDocument(test_document.TestPyDocument, BaseSoledadTest): - - scenarios = ([( - 'leap', {'make_document_for_test': make_leap_document_for_test})]) - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_remote_sync_target`. -#----------------------------------------------------------------------------- - -class TestSoledadSyncTargetBasics( - test_remote_sync_target.TestHTTPSyncTargetBasics): - """ - Some tests had to be copied to this class so we can instantiate our own - target. - """ - - def test_parse_url(self): - remote_target = target.SoledadSyncTarget('http://127.0.0.1:12345/') - self.assertEqual('http', remote_target._url.scheme) - self.assertEqual('127.0.0.1', remote_target._url.hostname) - self.assertEqual(12345, remote_target._url.port) - self.assertEqual('/', remote_target._url.path) - - -class TestSoledadParsingSyncStream( - test_remote_sync_target.TestParsingSyncStream, - BaseSoledadTest): - """ - Some tests had to be copied to this class so we can instantiate our own - target. - """ - - def setUp(self): - test_remote_sync_target.TestParsingSyncStream.setUp(self) - - def tearDown(self): - test_remote_sync_target.TestParsingSyncStream.tearDown(self) - - def test_extra_comma(self): - """ - Test adapted to use encrypted content. - """ - doc = SoledadDocument('i', rev='r') - doc.content = {} - enc_json = target.encrypt_doc(self._soledad._crypto, doc) - tgt = target.SoledadSyncTarget( - "http://foo/foo", crypto=self._soledad._crypto) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "[\r\n{},\r\n]", None) - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, - '[\r\n{},\r\n{"id": "i", "rev": "r", ' - '"content": %s, "gen": 3, "trans_id": "T-sid"}' - ',\r\n]' % json.dumps(enc_json), - lambda doc, gen, trans_id: None) - - def test_wrong_start(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "{}\r\n]", None) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "\r\n{}\r\n]", None) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "", None) - - def test_wrong_end(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "[\r\n{}", None) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "[\r\n", None) - - def test_missing_comma(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, - '[\r\n{}\r\n{"id": "i", "rev": "r", ' - '"content": "c", "gen": 3}\r\n]', None) - - def test_no_entries(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, "[\r\n]", None) - - def test_error_in_stream(self): - tgt = target.SoledadSyncTarget("http://foo/foo") - - self.assertRaises(u1db.errors.Unavailable, - tgt._parse_sync_stream, - '[\r\n{"new_generation": 0},' - '\r\n{"error": "unavailable"}\r\n', None) - - self.assertRaises(u1db.errors.Unavailable, - tgt._parse_sync_stream, - '[\r\n{"error": "unavailable"}\r\n', None) - - self.assertRaises(u1db.errors.BrokenSyncStream, - tgt._parse_sync_stream, - '[\r\n{"error": "?"}\r\n', None) - - -# -# functions for TestRemoteSyncTargets -# - -def leap_sync_target(test, path): - return target.SoledadSyncTarget( - test.getURL(path), crypto=test._soledad._crypto) - - -def token_leap_sync_target(test, path): - st = leap_sync_target(test, path) - st.set_token_credentials('user-uuid', 'auth-token') - return st - - -def make_local_db_and_soledad_target(test, path='test'): - test.startServer() - db = test.request_state._create_database(os.path.basename(path)) - st = target.SoledadSyncTarget.connect( - test.getURL(path), crypto=test._soledad._crypto) - return db, st - - -def make_local_db_and_token_soledad_target(test): - db, st = make_local_db_and_soledad_target(test, 'test') - st.set_token_credentials('user-uuid', 'auth-token') - return db, st - - -class TestSoledadSyncTarget( - SoledadWithCouchServerMixin, - test_remote_sync_target.TestRemoteSyncTargets): - - scenarios = [ - ('token_soledad', - {'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_leap_document_for_test, - 'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_database_for_test': make_sqlcipher_database_for_test, - 'sync_target': token_leap_sync_target}), - ] - - def setUp(self): - tests.TestCaseWithServer.setUp(self) - self.main_test_class = test_remote_sync_target.TestRemoteSyncTargets - SoledadWithCouchServerMixin.setUp(self) - self.startServer() - self.db1 = make_sqlcipher_database_for_test(self, 'test1') - self.db2 = self.request_state._create_database('test2') - - def tearDown(self): - SoledadWithCouchServerMixin.tearDown(self) - tests.TestCaseWithServer.tearDown(self) - db, _ = self.request_state.ensure_database('test2') - db.delete_database() - for i in ['db1', 'db2']: - if hasattr(self, i): - db = getattr(self, i) - db.close() - - def test_sync_exchange_send(self): - """ - Test for sync exchanging send of document. - - This test was adapted to decrypt remote content before assert. - """ - self.startServer() - db = self.request_state._create_database('test') - remote_target = self.getSyncTarget('test') - other_docs = [] - - def receive_doc(doc, gen, trans_id): - other_docs.append((doc.doc_id, doc.rev, doc.get_json())) - - doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - new_gen, trans_id = remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=receive_doc) - self.assertEqual(1, new_gen) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', False) - db.close() - - def test_sync_exchange_send_failure_and_retry_scenario(self): - """ - Test for sync exchange failure and retry. - - This test was adapted to: - - decrypt remote content before assert. - - not expect a bounced document because soledad has stateful - recoverable sync. - """ - - self.startServer() - - def blackhole_getstderr(inst): - return cStringIO.StringIO() - - self.patch(self.server.RequestHandlerClass, 'get_stderr', - blackhole_getstderr) - db = self.request_state._create_database('test') - _put_doc_if_newer = db._put_doc_if_newer - trigger_ids = ['doc-here2'] - - def bomb_put_doc_if_newer(self, doc, save_conflict, - replica_uid=None, replica_gen=None, - replica_trans_id=None, number_of_docs=None, - doc_idx=None, sync_id=None): - if doc.doc_id in trigger_ids: - raise Exception - return _put_doc_if_newer(doc, save_conflict=save_conflict, - replica_uid=replica_uid, - replica_gen=replica_gen, - replica_trans_id=replica_trans_id, - number_of_docs=number_of_docs, - doc_idx=doc_idx, - sync_id=sync_id) - from leap.soledad.common.tests.test_couch import IndexedCouchDatabase - self.patch( - IndexedCouchDatabase, '_put_doc_if_newer', bomb_put_doc_if_newer) - remote_target = self.getSyncTarget('test') - other_changes = [] - - def receive_doc(doc, gen, trans_id): - other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - doc2 = self.make_document('doc-here2', 'replica:1', - '{"value": "here2"}') - # We do not expect an exception here because the sync fails gracefully - remote_target.sync_exchange( - [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], - 'replica', last_known_generation=0, last_known_trans_id=None, - return_doc_cb=receive_doc) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', - False) - self.assertEqual( - (10, 'T-sid'), db._get_replica_gen_and_trans_id('replica')) - self.assertEqual([], other_changes) - # retry - trigger_ids = [] - new_gen, trans_id = remote_target.sync_exchange( - [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=receive_doc) - self.assertGetEncryptedDoc( - db, 'doc-here2', 'replica:1', '{"value": "here2"}', - False) - self.assertEqual( - (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica')) - self.assertEqual(2, new_gen) - self.assertEqual( - ('doc-here', 'replica:1', '{"value": "here"}', 1), - other_changes[0][:-1]) - db.close() - - def test_sync_exchange_send_ensure_callback(self): - """ - Test for sync exchange failure and retry. - - This test was adapted to decrypt remote content before assert. - """ - self.startServer() - remote_target = self.getSyncTarget('test') - other_docs = [] - replica_uid_box = [] - - def receive_doc(doc, gen, trans_id): - other_docs.append((doc.doc_id, doc.rev, doc.get_json())) - - def ensure_cb(replica_uid): - replica_uid_box.append(replica_uid) - - doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - new_gen, trans_id = remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=receive_doc, - ensure_callback=ensure_cb) - self.assertEqual(1, new_gen) - db = self.request_state.open_database('test') - self.assertEqual(1, len(replica_uid_box)) - self.assertEqual(db._replica_uid, replica_uid_box[0]) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', False) - db.close() - - def test_sync_exchange_in_stream_error(self): - # we bypass this test because our sync_exchange process does not - # return u1db error 503 "unavailable" for now. - pass - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_https`. -#----------------------------------------------------------------------------- - -def token_leap_https_sync_target(test, host, path): - _, port = test.server.server_address - st = target.SoledadSyncTarget( - 'https://%s:%d/%s' % (host, port, path), - crypto=test._soledad._crypto) - st.set_token_credentials('user-uuid', 'auth-token') - return st - - -class TestSoledadSyncTargetHttpsSupport( - test_https.TestHttpSyncTargetHttpsSupport, - BaseSoledadTest): - - scenarios = [ - ('token_soledad_https', - {'server_def': test_https.https_server_def, - 'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_leap_document_for_test, - 'sync_target': token_leap_https_sync_target}), - ] - - def setUp(self): - # the parent constructor undoes our SSL monkey patch to ensure tests - # run smoothly with standard u1db. - test_https.TestHttpSyncTargetHttpsSupport.setUp(self) - # so here monkey patch again to test our functionality. - http_client._VerifiedHTTPSConnection = VerifiedHTTPSConnection - client.SOLEDAD_CERT = http_client.CA_CERTS - - def test_working(self): - """ - Test that SSL connections work well. - - This test was adapted to patch Soledad's HTTPS connection custom class - with the intended CA certificates. - """ - self.startServer() - db = self.request_state._create_database('test') - self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) - remote_target = self.getSyncTarget('localhost', 'test') - remote_target.record_sync_info('other-id', 2, 'T-id') - self.assertEqual( - (2, 'T-id'), db._get_replica_gen_and_trans_id('other-id')) - - def test_host_mismatch(self): - """ - Test that SSL connections to a hostname different than the one in the - certificate raise CertificateError. - - This test was adapted to patch Soledad's HTTPS connection custom class - with the intended CA certificates. - """ - self.startServer() - self.request_state._create_database('test') - self.patch(client, 'SOLEDAD_CERT', self.cacert_pem) - remote_target = self.getSyncTarget('127.0.0.1', 'test') - self.assertRaises( - http_client.CertificateError, remote_target.record_sync_info, - 'other-id', 2, 'T-id') - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_database`. -#----------------------------------------------------------------------------- - -class _HTTPDatabase(http_database.HTTPDatabase, auth.TokenBasedAuth): - """ - Wraps our token auth implementation. - """ - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - -class TestHTTPDatabaseWithCreds( - test_http_database.TestHTTPDatabaseCtrWithCreds): - - def test_get_sync_target_inherits_token_credentials(self): - # this test was from TestDatabaseSimpleOperations but we put it here - # for convenience. - self.db = _HTTPDatabase('dbase') - self.db.set_token_credentials('user-uuid', 'auth-token') - st = self.db.get_sync_target() - self.assertEqual(self.db._creds, st._creds) - - def test_ctr_with_creds(self): - db1 = _HTTPDatabase('http://dbs/db', creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - self.assertIn('token', db1._creds) - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -#----------------------------------------------------------------------------- - -target_scenarios = [ - ('token_leap', {'create_db_and_target': - make_local_db_and_token_soledad_target, - 'make_app_with_state': make_soledad_app}), -] - - -class SoledadDatabaseSyncTargetTests( - SoledadWithCouchServerMixin, test_sync.DatabaseSyncTargetTests): - - scenarios = ( - tests.multiply_scenarios( - tests.DatabaseBaseTests.scenarios, - target_scenarios)) - - whitebox = False - - def setUp(self): - self.main_test_class = test_sync.DatabaseSyncTargetTests - SoledadWithCouchServerMixin.setUp(self) - - def test_sync_exchange(self): - """ - Test sync exchange. - - This test was adapted to decrypt remote content before assert. - """ - sol, _ = make_local_db_and_soledad_target(self) - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, - 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertTransactionLog(['doc-id'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, last_trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - sol.close() - - def test_sync_exchange_push_many(self): - """ - Test sync exchange. - - This test was adapted to decrypt remote content before assert. - """ - docs_by_gen = [ - (self.make_document( - 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), - (self.make_document( - 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) - self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(11, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_returns_many_new_docs(self): - """ - Test sync exchange. - - This test was adapted to avoid JSON serialization comparison as local - and remote representations might differ. It looks directly at the - doc's contents instead. - """ - doc = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db.create_doc_from_json(tests.nested_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - self.assertEqual(2, new_gen) - self.assertEqual( - [(doc.doc_id, doc.rev, 1), - (doc2.doc_id, doc2.rev, 2)], - [c[:-3] + c[-2:-1] for c in self.other_changes]) - self.assertEqual( - json.loads(tests.simple_doc), - json.loads(self.other_changes[0][2])) - self.assertEqual( - json.loads(tests.nested_doc), - json.loads(self.other_changes[1][2])) - if self.whitebox: - self.assertEqual( - self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': - [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - - -class TestSoledadDbSync( - SoledadWithCouchServerMixin, test_sync.TestDbSync): - """Test db.sync remote sync shortcut""" - - scenarios = [ - ('py-token-http', { - 'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_app_with_state': make_token_soledad_app, - 'make_database_for_test': make_sqlcipher_database_for_test, - 'token': True - }), - ] - - oauth = False - token = False - - def setUp(self): - self.main_test_class = test_sync.TestDbSync - SoledadWithCouchServerMixin.setUp(self) - - def tearDown(self): - SoledadWithCouchServerMixin.tearDown(self) - self.db.close() - - def do_sync(self, target_name): - """ - Perform sync using SoledadSyncTarget and Token auth. - """ - if self.token: - extra = dict(creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - target_url = self.getURL(target_name) - return Synchronizer( - self.db, - target.SoledadSyncTarget( - target_url, - crypto=self._soledad._crypto, - **extra)).sync(autocreate=True) - else: - return test_sync.TestDbSync.do_sync(self, target_name) - - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - doc1 = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = self.do_sync('test2') - gen, _, changes = self.db.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) - - def test_db_sync_autocreate(self): - """ - We bypass this test because we never need to autocreate databases. - """ - pass - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/test_target_soledad.py b/common/src/leap/soledad/common/tests/test_target_soledad.py deleted file mode 100644 index 899203b8..00000000 --- a/common/src/leap/soledad/common/tests/test_target_soledad.py +++ /dev/null @@ -1,102 +0,0 @@ -from u1db.remote import ( - http_database, -) - -from leap.soledad.client import ( - auth, - VerifiedHTTPSConnection, -) -from leap.soledad.common.document import SoledadDocument -from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests.u1db_tests import test_backends - - -#----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -#----------------------------------------------------------------------------- - -def make_leap_document_for_test(test, doc_id, rev, content, - has_conflicts=False): - return SoledadDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -def make_soledad_app(state): - return SoledadApp(state) - - -def make_token_soledad_app(state): - app = SoledadApp(state) - - def _verify_authentication_data(uuid, auth_data): - if uuid == 'user-uuid' and auth_data == 'auth-token': - return True - return False - - # we test for action authorization in leap.soledad.common.tests.test_server - def _verify_authorization(uuid, environ): - return True - - application = SoledadTokenAuthMiddleware(app) - application._verify_authentication_data = _verify_authentication_data - application._verify_authorization = _verify_authorization - return application - - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -def make_token_http_database_for_test(test, replica_uid): - test.startServer() - test.request_state._create_database(replica_uid) - - class _HTTPDatabaseWithToken( - http_database.HTTPDatabase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - http_db = _HTTPDatabaseWithToken(test.getURL('test')) - http_db.set_token_credentials('user-uuid', 'auth-token') - return http_db - - -def copy_token_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') - return http_db - - -class SoledadTests(test_backends.AllDatabaseTests, BaseSoledadTest): - - scenarios = LEAP_SCENARIOS + [ - ('token_http', {'make_database_for_test': - make_token_http_database_for_test, - 'copy_database_for_test': - copy_token_http_database_for_test, - 'make_document_for_test': make_leap_document_for_test, - 'make_app_with_state': make_token_soledad_app, - }) - ] - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py index ad66fb06..6efeb87f 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py @@ -35,7 +35,7 @@ from pysqlcipher import dbapi2 from StringIO import StringIO import testscenarios -import testtools +from twisted.trial import unittest from u1db import ( errors, @@ -50,7 +50,7 @@ from u1db.remote import ( ) -class TestCase(testtools.TestCase): +class TestCase(unittest.TestCase): def createTempDir(self, prefix='u1db-tmp-'): """Create a temporary directory to do some work in. diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py index c086fbc0..c5b316ed 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py @@ -75,7 +75,7 @@ class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): # order to maintain the compatibility with u1db default tests, we undo # that replacement here. http_client._VerifiedHTTPSConnection = \ - soledad.client.old__VerifiedHTTPSConnection + soledad.client.api.old__VerifiedHTTPSConnection super(TestHttpSyncTargetHttpsSupport, self).setUp() def getSyncTarget(self, host, path=None): diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py index b27e6356..d4439ef4 100644 --- a/common/src/leap/soledad/common/tests/util.py +++ b/common/src/leap/soledad/common/tests/util.py @@ -21,24 +21,40 @@ Utilities used by multiple test suites. """ +import os import tempfile import shutil +import random +import string +import u1db +import subprocess +import time +import re + +from mock import Mock from urlparse import urljoin - from StringIO import StringIO from pysqlcipher import dbapi2 + from u1db.errors import DatabaseDoesNotExist +from u1db.remote import http_database + +from twisted.trial import unittest +from leap.common.files import mkdir_p from leap.soledad.common import soledad_assert +from leap.soledad.common.document import SoledadDocument from leap.soledad.common.couch import CouchDatabase, CouchServerState -from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.client import Soledad +from leap.soledad.client import target +from leap.soledad.client import auth +from leap.soledad.client.crypto import decrypt_doc_dict -from leap.soledad.common.tests import BaseSoledadTest -from leap.soledad.common.tests.test_couch import CouchDBWrapper, CouchDBTestCase - +from leap.soledad.server import SoledadApp +from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.client.sqlcipher import ( SQLCipherDatabase, @@ -47,6 +63,7 @@ from leap.soledad.client.sqlcipher import ( PASSWORD = '123456' +ADDRESS = 'leap@leap.se' def make_sqlcipher_database_for_test(test, replica_uid): @@ -62,7 +79,7 @@ def copy_sqlcipher_database_for_test(test, db): # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR # HOUSE. - new_db = SQLCipherDatabase(':memory:', PASSWORD) + new_db = make_sqlcipher_database_for_test(test, None) tmpfile = StringIO() for line in db._db_handle.iterdump(): if not 'sqlite_sequence' in line: # work around bug in iterdump @@ -98,6 +115,295 @@ def make_token_soledad_app(state): return application +def make_soledad_document_for_test(test, doc_id, rev, content, + has_conflicts=False): + return SoledadDocument( + doc_id, rev, content, has_conflicts=has_conflicts) + + +def make_token_http_database_for_test(test, replica_uid): + test.startServer() + test.request_state._create_database(replica_uid) + + class _HTTPDatabaseWithToken( + http_database.HTTPDatabase, auth.TokenBasedAuth): + + def set_token_credentials(self, uuid, token): + auth.TokenBasedAuth.set_token_credentials(self, uuid, token) + + def _sign_request(self, method, url_query, params): + return auth.TokenBasedAuth._sign_request( + self, method, url_query, params) + + http_db = _HTTPDatabaseWithToken(test.getURL('test')) + http_db.set_token_credentials('user-uuid', 'auth-token') + return http_db + + +def copy_token_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + http_db = test.request_state._copy_database(db) + http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') + return http_db + + +class MockedSharedDBTest(object): + + def get_default_shared_mock(self, put_doc_side_effect=None, + get_doc_return_value=None): + """ + Get a default class for mocking the shared DB + """ + class defaultMockSharedDB(object): + get_doc = Mock(return_value=get_doc_return_value) + put_doc = Mock(side_effect=put_doc_side_effect) + lock = Mock(return_value=('atoken', 300)) + unlock = Mock(return_value=True) + open = Mock(return_value=None) + syncable = True + + def __call__(self): + return self + return defaultMockSharedDB + + +def soledad_sync_target(test, path): + return target.SoledadSyncTarget( + test.getURL(path), crypto=test._soledad._crypto) + + +def token_soledad_sync_target(test, path): + st = soledad_sync_target(test, path) + st.set_token_credentials('user-uuid', 'auth-token') + return st + + +class BaseSoledadTest(unittest.TestCase, MockedSharedDBTest): + """ + Instantiates Soledad for usage in tests. + """ + defer_sync_encryption = False + + def setUp(self): + # The following snippet comes from BaseLeapTest.setUpClass, but we + # repeat it here because twisted.trial does not work with + # setUpClass/tearDownClass. + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + self.db2_file = os.path.join(self.tempdir, "db2.u1db") + self.email = ADDRESS + # open test dbs + self._db1 = u1db.open(self.db1_file, create=True, + document_factory=SoledadDocument) + self._db2 = u1db.open(self.db2_file, create=True, + document_factory=SoledadDocument) + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + # initialize soledad by hand so we can control keys + # XXX check if this soledad is actually used + self._soledad = self._soledad_instance( + prefix=self.rand_prefix, user=self.email) + + def tearDown(self): + self._db1.close() + self._db2.close() + self._soledad.close() + + # restore paths + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + + def _delete_temporary_dirs(): + # XXX should not access "private" attrs + for f in [self._soledad.local_db_path, + self._soledad.secrets.secrets_path]: + if os.path.isfile(f): + os.unlink(f) + # The following snippet comes from BaseLeapTest.setUpClass, but we + # repeat it here because twisted.trial does not work with + # setUpClass/tearDownClass. + soledad_assert( + self.tempdir.startswith('/tmp/leap_tests-'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(self.tempdir) + + from twisted.internet import reactor + reactor.addSystemEventTrigger( + "after", "shutdown", _delete_temporary_dirs) + + + def _soledad_instance(self, user=ADDRESS, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', + server_url='https://127.0.0.1/', + cert_file=None, + shared_db_class=None, + auth_token='auth-token'): + + def _put_doc_side_effect(doc): + self._doc_put = doc + + if shared_db_class is not None: + MockSharedDB = shared_db_class + else: + MockSharedDB = self.get_default_shared_mock( + _put_doc_side_effect) + + return Soledad( + user, + passphrase, + secrets_path=os.path.join( + self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, # Soledad will fail if not given an url. + cert_file=cert_file, + defer_encryption=self.defer_sync_encryption, + shared_db=MockSharedDB(), + auth_token=auth_token) + + def assertGetEncryptedDoc( + self, db, doc_id, doc_rev, content, has_conflicts): + """ + Assert that the document in the database looks correct. + """ + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + doc = db.get_doc(doc_id) + + if ENC_SCHEME_KEY in doc.content: + # XXX check for SYM_KEY too + key = self._soledad._crypto.doc_passphrase(doc.doc_id) + secret = self._soledad._crypto.secret + decrypted = decrypt_doc_dict( + doc.content, doc.doc_id, doc.rev, + key, secret) + doc.set_json(decrypted) + self.assertEqual(exp_doc.doc_id, doc.doc_id) + self.assertEqual(exp_doc.rev, doc.rev) + self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts) + self.assertEqual(exp_doc.content, doc.content) + + +#----------------------------------------------------------------------------- +# A wrapper for running couchdb locally. +#----------------------------------------------------------------------------- + +# from: https://github.com/smcq/paisley/blob/master/paisley/test/util.py +# TODO: include license of above project. +class CouchDBWrapper(object): + """ + Wrapper for external CouchDB instance which is started and stopped for + testing. + """ + + def start(self): + """ + Start a CouchDB instance for a test. + """ + self.tempdir = tempfile.mkdtemp(suffix='.couch.test') + + path = os.path.join(os.path.dirname(__file__), + 'couchdb.ini.template') + handle = open(path) + conf = handle.read() % { + 'tempdir': self.tempdir, + } + handle.close() + + confPath = os.path.join(self.tempdir, 'test.ini') + handle = open(confPath, 'w') + handle.write(conf) + handle.close() + + # create the dirs from the template + mkdir_p(os.path.join(self.tempdir, 'lib')) + mkdir_p(os.path.join(self.tempdir, 'log')) + args = ['/usr/bin/couchdb', '-n', '-a', confPath] + null = open('/dev/null', 'w') + + self.process = subprocess.Popen( + args, env=None, stdout=null.fileno(), stderr=null.fileno(), + close_fds=True) + # find port + logPath = os.path.join(self.tempdir, 'log', 'couch.log') + while not os.path.exists(logPath): + if self.process.poll() is not None: + got_stdout, got_stderr = "", "" + if self.process.stdout is not None: + got_stdout = self.process.stdout.read() + + if self.process.stderr is not None: + got_stderr = self.process.stderr.read() + raise Exception(""" +couchdb exited with code %d. +stdout: +%s +stderr: +%s""" % ( + self.process.returncode, got_stdout, got_stderr)) + time.sleep(0.01) + while os.stat(logPath).st_size == 0: + time.sleep(0.01) + PORT_RE = re.compile( + 'Apache CouchDB has started on http://127.0.0.1:(?P\d+)') + + handle = open(logPath) + line = handle.read() + handle.close() + m = PORT_RE.search(line) + if not m: + self.stop() + raise Exception("Cannot find port in line %s" % line) + self.port = int(m.group('port')) + + def stop(self): + """ + Terminate the CouchDB instance. + """ + self.process.terminate() + self.process.communicate() + shutil.rmtree(self.tempdir) + + +class CouchDBTestCase(unittest.TestCase, MockedSharedDBTest): + """ + TestCase base class for tests against a real CouchDB server. + """ + + def setUp(self): + """ + Make sure we have a CouchDB instance for a test. + """ + self.wrapper = CouchDBWrapper() + self.wrapper.start() + #self.db = self.wrapper.db + + def tearDown(self): + """ + Stop CouchDB instance for test. + """ + self.wrapper.stop() + class CouchServerStateForTests(CouchServerState): """ This is a slightly modified CouchDB server state that allows for creating @@ -126,43 +432,15 @@ class SoledadWithCouchServerMixin( BaseSoledadTest, CouchDBTestCase): - @classmethod - def setUpClass(cls): - """ - Make sure we have a CouchDB instance for a test. - """ - # from BaseLeapTest - cls.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - # from CouchDBTestCase - cls.wrapper = CouchDBWrapper() - cls.wrapper.start() - #self.db = self.wrapper.db - - @classmethod - def tearDownClass(cls): - """ - Stop CouchDB instance for test. - """ - # from BaseLeapTest - soledad_assert( - cls.tempdir.startswith('/tmp/leap_tests-'), - "beware! tried to remove a dir which does not " - "live in temporal folder!") - shutil.rmtree(cls.tempdir) - # from CouchDBTestCase - cls.wrapper.stop() - def setUp(self): - BaseSoledadTest.setUp(self) CouchDBTestCase.setUp(self) + BaseSoledadTest.setUp(self) main_test_class = getattr(self, 'main_test_class', None) if main_test_class is not None: main_test_class.setUp(self) self._couch_url = 'http://localhost:%d' % self.wrapper.port def tearDown(self): - BaseSoledadTest.tearDown(self) - CouchDBTestCase.tearDown(self) main_test_class = getattr(self, 'main_test_class', None) if main_test_class is not None: main_test_class.tearDown(self) @@ -172,6 +450,8 @@ class SoledadWithCouchServerMixin( db.delete_database() except DatabaseDoesNotExist: pass + BaseSoledadTest.tearDown(self) + CouchDBTestCase.tearDown(self) def make_app(self): couch_url = urljoin( -- cgit v1.2.3 From 279174f1e087e8f52767f8a76cb98c73fe614239 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 14:59:10 -0200 Subject: Cleanup unused import. --- common/src/leap/soledad/common/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py index 23d28e76..c5c4b97f 100644 --- a/common/src/leap/soledad/common/__init__.py +++ b/common/src/leap/soledad/common/__init__.py @@ -21,14 +21,10 @@ Soledad routines common to client and server. """ -from hashlib import sha256 - - # # Global constants # - SHARED_DB_NAME = 'shared' SHARED_DB_LOCK_DOC_ID_PREFIX = 'lock-' USER_DB_PREFIX = 'user-' -- cgit v1.2.3 From df28f2f99248bdff1a1704e9f6afff7e063d30e9 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 15:38:27 -0200 Subject: Several fixes in soledad api. * Allow passing shared_db to Soledad constructor. * Close syncers on Soledad close. * Fix docstrings. --- client/src/leap/soledad/client/api.py | 373 +++++++++++++++++++++++++++++++--- 1 file changed, 340 insertions(+), 33 deletions(-) diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 00884a12..59cbc4ca 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -46,6 +46,7 @@ from zope.interface import implements from twisted.python import log from leap.common.config import get_path_prefix + from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type @@ -112,7 +113,7 @@ class Soledad(object): default_prefix = os.path.join(get_path_prefix(), 'leap', 'soledad') def __init__(self, uuid, passphrase, secrets_path, local_db_path, - server_url, cert_file, + server_url, cert_file, shared_db=None, auth_token=None, defer_encryption=False, syncable=True): """ Initialize configuration, cryptographic keys and dbs. @@ -142,6 +143,10 @@ class Soledad(object): certificate used by the remote soledad server. :type cert_file: str + :param shared_db: + The shared database. + :type shared_db: HTTPDatabase + :param auth_token: Authorization token for accessing remote databases. :type auth_token: str @@ -157,8 +162,9 @@ class Soledad(object): :type syncable: bool :raise BootstrapSequenceError: - Raised when the secret generation and storage on server sequence - has failed for some reason. + Raised when the secret initialization sequence (i.e. retrieval + from server or generation and storage on server) has failed for + some reason. """ # store config params self._uuid = uuid @@ -168,7 +174,7 @@ class Soledad(object): self._defer_encryption = defer_encryption self._secrets_path = None - self.shared_db = None + self.shared_db = shared_db # configure SSL certificate global SOLEDAD_CERT @@ -225,6 +231,9 @@ class Soledad(object): create_path_if_not_exists(path) def _init_secrets(self): + """ + Initialize Soledad secrets. + """ self._secrets = SoledadSecrets( self.uuid, self._passphrase, self._secrets_path, self.shared_db, self._crypto) @@ -232,8 +241,9 @@ class Soledad(object): def _init_u1db_sqlcipher_backend(self): """ - Initialize the U1DB SQLCipher database for local storage, by - instantiating a modified twisted adbapi that will maintain a threadpool + Initialize the U1DB SQLCipher database for local storage. + + Instantiates a modified twisted adbapi that will maintain a threadpool with a u1db-sqclipher connection for each thread, and will return deferreds for each u1db query. @@ -253,14 +263,16 @@ class Soledad(object): defer_encryption=self._defer_encryption, sync_db_key=sync_db_key, ) - self._soledad_opts = opts + self._sqlcipher_opts = opts self._dbpool = adbapi.getConnectionPool(opts) def _init_u1db_syncer(self): + """ + Initialize the U1DB synchronizer. + """ replica_uid = self._dbpool.replica_uid - print "replica UID (syncer init)", replica_uid self._dbsyncer = SQLCipherU1DBSync( - self._soledad_opts, self._crypto, replica_uid, + self._sqlcipher_opts, self._crypto, replica_uid, self._defer_encryption) # @@ -273,99 +285,351 @@ class Soledad(object): """ logger.debug("Closing soledad") self._dbpool.close() - - # TODO close syncers >>>>>> + self._dbsyncer.close() # # ILocalStorage # def _defer(self, meth, *args, **kw): + """ + Defer a method to be run on a U1DB connection pool. + + :param meth: A method to defer to the U1DB connection pool. + :type meth: callable + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._dbpool.runU1DBQuery(meth, *args, **kw) def put_doc(self, doc): """ + Update a document. + + If the document currently has conflicts, put will fail. + If the database specifies a maximum document size and the document + exceeds it, put will fail and raise a DocumentTooBig exception. + ============================== WARNING ============================== This method converts the document's contents to unicode in-place. This means that after calling `put_doc(doc)`, the contents of the document, i.e. `doc.content`, might be different from before the call. ============================== WARNING ============================== + + :param doc: A document with new content. + :type doc: leap.soledad.common.document.SoledadDocument + :return: A deferred whose callback will be invoked with the new + revision identifier for the document. The document object will + also be updated. + :rtype: twisted.internet.defer.Deferred """ doc.content = _convert_to_unicode(doc.content) return self._defer("put_doc", doc) def delete_doc(self, doc): - # XXX what does this do when fired??? + """ + Mark a document as deleted. + + Will abort if the current revision doesn't match doc.rev. + This will also set doc.content to None. + + :param doc: A document to be deleted. + :type doc: leap.soledad.common.document.SoledadDocument + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("delete_doc", doc) def get_doc(self, doc_id, include_deleted=False): + """ + Get the JSON string for the given document. + + :param doc_id: The unique document identifier + :type doc_id: str + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise asking for a deleted + document will return None. + :type include_deleted: bool + :return: A deferred whose callback will be invoked with a document + object. + :rtype: twisted.internet.defer.Deferred + """ return self._defer( "get_doc", doc_id, include_deleted=include_deleted) def get_docs( self, doc_ids, check_for_conflicts=True, include_deleted=False): + """ + Get the JSON content for many documents. + + :param doc_ids: A list of document identifiers. + :type doc_ids: list + :param check_for_conflicts: If set to False, then the conflict check + will be skipped, and 'None' will be returned instead of True/False. + :type check_for_conflicts: bool + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :type include_deleted: bool + :return: A deferred whose callback will be invoked with an iterable + giving the document object for each document id in matching + doc_ids order. + :rtype: twisted.internet.defer.Deferred + """ return self._defer( "get_docs", doc_ids, check_for_conflicts=check_for_conflicts, include_deleted=include_deleted) def get_all_docs(self, include_deleted=False): + """ + Get the JSON content for all documents in the database. + + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :type include_deleted: bool + + :return: A deferred which, when fired, will pass the a tuple + containing (generation, [Document]) to the callback, with the + current generation of the database, followed by a list of all the + documents in the database. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_all_docs", include_deleted) def create_doc(self, content, doc_id=None): + """ + Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param content: A Python dictionary. + :type content: dict + :param doc_id: An optional identifier specifying the document id. + :type doc_id: str + :return: A deferred whose callback will be invoked with a document. + :rtype: twisted.internet.defer.Deferred + """ return self._defer( "create_doc", _convert_to_unicode(content), doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): + """ + Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param json: The JSON document string + :type json: dict + :param doc_id: An optional identifier specifying the document id. + :type doc_id: str + :return: A deferred whose callback will be invoked with a document. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("create_doc_from_json", json, doc_id=doc_id) def create_index(self, index_name, *index_expressions): + """ + Create a named index, which can then be queried for future lookups. + + Creating an index which already exists is not an error, and is cheap. + Creating an index which does not match the index_expressions of the + existing index is an error. + Creating an index will block until the expressions have been evaluated + and the index generated. + + :param index_name: A unique name which can be used as a key prefix + :type index_name: str + :param index_expressions: index expressions defining the index + information. + + Examples: + + "fieldname", or "fieldname.subfieldname" to index alphabetically + sorted on the contents of a field. + + "number(fieldname, width)", "lower(fieldname)" + :type index_expresions: list of str + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("create_index", index_name, *index_expressions) def delete_index(self, index_name): + """ + Remove a named index. + + :param index_name: The name of the index we are removing + :type index_name: str + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("delete_index", index_name) def list_indexes(self): + """ + List the definitions of all known indexes. + + :return: A deferred whose callback will be invoked with a list of + [('index-name', ['field', 'field2'])] definitions. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("list_indexes") def get_from_index(self, index_name, *key_values): + """ + Return documents that match the keys supplied. + + You must supply exactly the same number of values as have been defined + in the index. It is possible to do a prefix match by using '*' to + indicate a wildcard match. You can only supply '*' to trailing entries, + (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) + It is also possible to append a '*' to the last supplied value (eg + 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: list + :return: A deferred whose callback will be invoked with a list of + [Document]. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_from_index", index_name, *key_values) def get_count_from_index(self, index_name, *key_values): + """ + Return the count for a given combination of index_name + and key values. + + Extension method made from similar methods in u1db version 13.09 + + :param index_name: The index to query + :type index_name: str + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :type key_values: tuple + :return: A deferred whose callback will be invoked with the count. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_count_from_index", index_name, *key_values) def get_range_from_index(self, index_name, start_value, end_value): + """ + Return documents that fall within the specified range. + + Both ends of the range are inclusive. For both start_value and + end_value, one must supply exactly the same number of values as have + been defined in the index, or pass None. In case of a single column + index, a string is accepted as an alternative for a tuple with a single + value. It is possible to do a prefix match by using '*' to indicate + a wildcard match. You can only supply '*' to trailing entries, (eg + 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also + possible to append a '*' to the last supplied value (eg 'val*', '*', + '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :type index_name: str + :param start_values: tuples of values that define the lower bound of + the range. eg, if you have an index with 3 fields then you would + have: (val1, val2, val3) + :type start_values: tuple + :param end_values: tuples of values that define the upper bound of the + range. eg, if you have an index with 3 fields then you would have: + (val1, val2, val3) + :type end_values: tuple + :return: A deferred whose callback will be invoked with a list of + [Document]. + :rtype: twisted.internet.defer.Deferred + """ + return self._defer( "get_range_from_index", index_name, start_value, end_value) def get_index_keys(self, index_name): + """ + Return all keys under which documents are indexed in this index. + + :param index_name: The index to query + :type index_name: str + :return: A deferred whose callback will be invoked with a list of + tuples of indexed keys. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_index_keys", index_name) def get_doc_conflicts(self, doc_id): + """ + Get the list of conflicts for the given document. + + The order of the conflicts is such that the first entry is the value + that would be returned by "get_doc". + + :param doc_id: The unique document identifier + :type doc_id: str + :return: A deferred whose callback will be invoked with a list of the + Document entries that are conflicted. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("get_doc_conflicts", doc_id) def resolve_doc(self, doc, conflicted_doc_revs): + """ + Mark a document as no longer conflicted. + + We take the list of revisions that the client knows about that it is + superseding. This may be a different list from the actual current + conflicts, in which case only those are removed as conflicted. This + may fail if the conflict list is significantly different from the + supplied information. (sync could have happened in the background from + the time you GET_DOC_CONFLICTS until the point where you RESOLVE) + + :param doc: A Document with the new content to be inserted. + :type doc: SoledadDocument + :param conflicted_doc_revs: A list of revisions that the new content + supersedes. + :type conflicted_doc_revs: list(str) + :return: A deferred. + :rtype: twisted.internet.defer.Deferred + """ return self._defer("resolve_doc", doc, conflicted_doc_revs) - def _get_local_db_path(self): + @property + def local_db_path(self): return self._local_db_path - # XXX Do we really need all this private / property dance? - - local_db_path = property( - _get_local_db_path, - doc='The path for the local database replica.') - - def _get_uuid(self): + @property + def uuid(self): return self._uuid - uuid = property(_get_uuid, doc='The user uuid.') - # # ISyncableStorage # def sync(self, defer_decryption=True): + """ + Synchronize documents with the server replica. + + :param defer_decryption: + Whether to defer decryption of documents, or do it inline while + syncing. + :type defer_decryption: bool + :return: A deferred whose callback will be invoked with the local + generation before the synchronization was performed. + :rtype: twisted.internet.defer.Deferred + """ # ----------------------------------------------------------------- # TODO this needs work. @@ -377,7 +641,6 @@ class Soledad(object): # thread) # (4) Check that the deferred is called with the local gen. - # TODO document that this returns a deferred # ----------------------------------------------------------------- def on_sync_done(local_gen): @@ -404,6 +667,12 @@ class Soledad(object): @property def syncing(self): + """ + Return wether Soledad is currently synchronizing with the server. + + :return: Wether Soledad is currently synchronizing with the server. + :rtype: bool + """ return self._dbsyncer.syncing def _set_token(self, token): @@ -413,10 +682,11 @@ class Soledad(object): Internally, this builds the credentials dictionary with the following format: - self._{ + { 'token': { 'uuid': '' 'token': '' + } } :param token: The authentication token. @@ -442,18 +712,38 @@ class Soledad(object): # def init_shared_db(self, server_url, uuid, creds, syncable=True): - shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) - self.shared_db = SoledadSharedDatabase.open_database( - shared_db_url, - uuid, - creds=creds, - create=False, # db should exist at this point. - syncable=syncable) + """ + Initialize the shared database. + + :param server_url: URL of the remote database. + :type server_url: str + :param uuid: The user's unique id. + :type uuid: str + :param creds: A tuple containing the authentication method and + credentials. + :type creds: tuple + :param syncable: + If syncable is False, the database will not attempt to sync against + a remote replica. + :type syncable: bool + """ + # only case this is False is for testing purposes + if self.shared_db is None: + shared_db_url = urlparse.urljoin(server_url, SHARED_DB_NAME) + self.shared_db = SoledadSharedDatabase.open_database( + shared_db_url, + uuid, + creds=creds, + create=False, # db should exist at this point. + syncable=syncable) @property def storage_secret(self): """ - Return the secret used for symmetric encryption. + Return the secret used for local storage encryption. + + :return: The secret used for local storage encryption. + :rtype: str """ return self._secrets.storage_secret @@ -461,20 +751,37 @@ class Soledad(object): def remote_storage_secret(self): """ Return the secret used for encryption of remotely stored data. + + :return: The secret used for remote storage encryption. + :rtype: str """ return self._secrets.remote_storage_secret @property def secrets(self): + """ + Return the secrets object. + + :return: The secrets object. + :rtype: SoledadSecrets + """ return self._secrets def change_passphrase(self, new_passphrase): + """ + Change the passphrase that encrypts the storage secret. + + :param new_passphrase: The new passphrase. + :type new_passphrase: unicode + + :raise NoStorageSecret: Raised if there's no storage secret available. + """ self._secrets.change_passphrase(new_passphrase) def _convert_to_unicode(content): """ - Convert content to unicode (or all the strings in content) + Convert content to unicode (or all the strings in content). NOTE: Even though this method supports any type, it will currently ignore contents of lists, tuple or any other -- cgit v1.2.3 From 5a3dee72a03cc930f3357a8ea2a0d6395fdaaab7 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 15:40:44 -0200 Subject: Several fixes in adbapi interface: * Get replica uid upon U1DBConnectionPool initialization. * Fix docstrings. --- client/src/leap/soledad/client/adbapi.py | 101 +++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 6 deletions(-) diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index 0cdc90eb..9ae2889e 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -37,6 +37,23 @@ if DEBUG_SQL: def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): + """ + Return a connection pool. + + :param opts: + Options for the SQLCipher connection. + :type opts: SQLCipherOptions + :param openfun: + Callback invoked after every connect() on the underlying DB-API + object. + :type openfun: callable + :param driver: + The connection driver. + :type driver: str + + :return: A U1DB connection pool. + :rtype: U1DBConnectionPool + """ if openfun is None and driver == "pysqlcipher": openfun = partial(soledad_sqlcipher.set_init_pragmas, opts=opts) return U1DBConnectionPool( @@ -45,14 +62,29 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): class U1DBConnection(adbapi.Connection): + """ + A wrapper for a U1DB connection instance. + """ u1db_wrapper = soledad_sqlcipher.SoledadSQLCipherWrapper + """ + The U1DB wrapper to use. + """ def __init__(self, pool, init_u1db=False): + """ + :param pool: The pool of connections to that owns this connection. + :type pool: adbapi.ConnectionPool + :param init_u1db: Wether the u1db database should be initialized. + :type init_u1db: bool + """ self.init_u1db = init_u1db adbapi.Connection.__init__(self, pool) def reconnect(self): + """ + Reconnect to the U1DB database. + """ if self._connection is not None: self._pool.disconnect(self._connection) self._connection = self._pool.connect() @@ -61,29 +93,51 @@ class U1DBConnection(adbapi.Connection): self._u1db = self.u1db_wrapper(self._connection) def __getattr__(self, name): + """ + Route the requested attribute either to the U1DB wrapper or to the + connection. + + :param name: The name of the attribute. + :type name: str + """ if name.startswith('u1db_'): - meth = re.sub('^u1db_', '', name) - return getattr(self._u1db, meth) + attr = re.sub('^u1db_', '', name) + return getattr(self._u1db, attr) else: return getattr(self._connection, name) - class U1DBTransaction(adbapi.Transaction): + """ + A wrapper for a U1DB 'cursor' object. + """ def __getattr__(self, name): + """ + Route the requested attribute either to the U1DB wrapper of the + connection or to the actual connection cursor. + + :param name: The name of the attribute. + :type name: str + """ if name.startswith('u1db_'): - meth = re.sub('^u1db_', '', name) - return getattr(self._connection._u1db, meth) + attr = re.sub('^u1db_', '', name) + return getattr(self._connection._u1db, attr) else: return getattr(self._cursor, name) class U1DBConnectionPool(adbapi.ConnectionPool): + """ + Represent a pool of connections to an U1DB database. + """ connectionFactory = U1DBConnection transactionFactory = U1DBTransaction def __init__(self, *args, **kwargs): + """ + Initialize the connection pool. + """ adbapi.ConnectionPool.__init__(self, *args, **kwargs) # all u1db connections, hashed by thread-id self._u1dbconnections = {} @@ -91,15 +145,48 @@ class U1DBConnectionPool(adbapi.ConnectionPool): # The replica uid, primed by the connections on init. self.replica_uid = ProxyBase(None) + conn = self.connectionFactory(self, init_u1db=True) + replica_uid = conn._u1db._real_replica_uid + setProxiedObject(self.replica_uid, replica_uid) + def runU1DBQuery(self, meth, *args, **kw): + """ + Execute a U1DB query in a thread, using a pooled connection. + + :param meth: The U1DB wrapper method name. + :type meth: str + + :return: a Deferred which will fire the return value of + 'self._runU1DBQuery(Transaction(...), *args, **kw)', or a Failure. + :rtype: twisted.internet.defer.Deferred + """ meth = "u1db_%s" % meth return self.runInteraction(self._runU1DBQuery, meth, *args, **kw) def _runU1DBQuery(self, trans, meth, *args, **kw): + """ + Execute a U1DB query. + + :param trans: An U1DB transaction. + :type trans: adbapi.Transaction + :param meth: the U1DB wrapper method name. + :type meth: str + """ meth = getattr(trans, meth) return meth(*args, **kw) def _runInteraction(self, interaction, *args, **kw): + """ + Interact with the database and return the result. + + :param interaction: + A callable object whose first argument is an + L{adbapi.Transaction}. + :type interaction: callable + :return: a Deferred which will fire the return value of + 'interaction(Transaction(...), *args, **kw)', or a Failure. + :rtype: twisted.internet.defer.Deferred + """ tid = self.threadID() u1db = self._u1dbconnections.get(tid) conn = self.connectionFactory(self, init_u1db=not bool(u1db)) @@ -107,7 +194,6 @@ class U1DBConnectionPool(adbapi.ConnectionPool): if self.replica_uid is None: replica_uid = conn._u1db._real_replica_uid setProxiedObject(self.replica_uid, replica_uid) - print "GOT REPLICA UID IN DBPOOL", self.replica_uid if u1db is None: self._u1dbconnections[tid] = conn._u1db @@ -129,6 +215,9 @@ class U1DBConnectionPool(adbapi.ConnectionPool): raise excType, excValue, excTraceback def finalClose(self): + """ + A final close, only called by the shutdown trigger. + """ self.shutdownID = None self.threadpool.stop() self.running = False -- cgit v1.2.3 From 8f01a07faa6abccfdc6face00e8b6f95b184abdf Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 15:42:29 -0200 Subject: Several fixes in Soledad crypto: * Adapt to removal of the old multiprocessing safe database, by directly querying the sync database. * Fix docstrings. --- client/src/leap/soledad/client/crypto.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index aa8135c0..950576ec 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -530,8 +530,8 @@ class SyncEncryptDecryptPool(object): :param crypto: A SoledadCryto instance to perform the encryption. :type crypto: leap.soledad.crypto.SoledadCrypto - :param sync_db: a database connection handle - :type sync_db: handle + :param sync_db: A database connection handle + :type sync_db: pysqlcipher.dbapi2.Connection :param write_lock: a write lock for controlling concurrent access to the sync_db @@ -909,8 +909,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): if encrypted is not None: sql += " WHERE encrypted = %d" % int(encrypted) sql += " ORDER BY gen ASC" - docs = self._sync_db.select(sql) - return docs + return self._fetchall(sql) def get_insertable_docs_by_gen(self): """ @@ -927,15 +926,12 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): decrypted_docs = self.get_docs_by_generation(encrypted=False) insertable = [] for doc_id, rev, _, gen, trans_id, encrypted in all_docs: - try: - next_doc_id, _, next_content, _, _, _ = decrypted_docs.next() + for next_doc_id, _, next_content, _, _, _ in decrypted_docs: if doc_id == next_doc_id: content = next_content insertable.append((doc_id, rev, content, gen, trans_id)) else: break - except StopIteration: - break return insertable def count_docs_in_sync_db(self, encrypted=None): @@ -955,9 +951,9 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): sql = "SELECT COUNT(*) FROM %s" % (self.TABLE_NAME,) if encrypted is not None: sql += " WHERE encrypted = %d" % int(encrypted) - res = self._sync_db.select(sql) - if res is not None: - val = res.next() + res = self._fetchall(sql) + if res: + val = res.pop() return val[0] else: return 0 @@ -1035,4 +1031,10 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): Empty the received docs table of the sync database. """ sql = "DELETE FROM %s WHERE 1" % (self.TABLE_NAME,) - res = self._sync_db.execute(sql) + self._sync_db.execute(sql) + + def _fetchall(self, *args, **kwargs): + with self._sync_db: + c = self._sync_db.cursor() + c.execute(*args, **kwargs) + return c.fetchall() -- cgit v1.2.3 From dac64ed7d4f9749a620dcbfcabd33e46a94da63c Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 15:54:21 -0200 Subject: Several fixes in SoledadSharedDB: * Remove check for HTTPS address. * Remove creation of shared database. * Fix docstrings. --- client/src/leap/soledad/client/api.py | 1 - client/src/leap/soledad/client/shared_db.py | 22 ++++++++-------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 59cbc4ca..998e9148 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -734,7 +734,6 @@ class Soledad(object): shared_db_url, uuid, creds=creds, - create=False, # db should exist at this point. syncable=syncable) @property diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index 77a7db68..26ddc285 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -95,9 +95,7 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): # @staticmethod - def open_database(url, uuid, create, creds=None, syncable=True): - # TODO: users should not be able to create the shared database, so we - # have to remove this from here in the future. + def open_database(url, uuid, creds=None, syncable=True): """ Open a Soledad shared database. @@ -105,12 +103,9 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :type url: str :param uuid: The user's unique id. :type uuid: str - :param create: Should the database be created if it does not already - exist? - :type create: bool - :param token: An authentication token for accessing the shared db. - :type token: str - + :param creds: A tuple containing the authentication method and + credentials. + :type creds: tuple :param syncable: If syncable is False, the database will not attempt to sync against a remote replica. @@ -119,13 +114,12 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :return: The shared database in the given url. :rtype: SoledadSharedDatabase """ - if syncable and not url.startswith('https://'): - raise ImproperlyConfiguredError( - "Remote soledad server must be an https URI") + # XXX fix below, doesn't work with tests. + #if syncable and not url.startswith('https://'): + # raise ImproperlyConfiguredError( + # "Remote soledad server must be an https URI") db = SoledadSharedDatabase(url, uuid, creds=creds) db.syncable = syncable - if syncable: - db.open(create) return db @staticmethod -- cgit v1.2.3 From 5247c231639fc9fd8c4279190af50afa99783bd6 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 16:11:53 -0200 Subject: Several fixes in SQLCipherDatabase: * Add copy of SQLCipherOptions object to avoid modifying the options object in place when instantiating the sync db. * Add string representation of SQLCipherOptions for easiness of debugging. * Make sync db always "ready". * Fix passing options for sync db initialization. * Fix typ0 that made SQLCipherU1DBSync._sync_loop be a tuple. * Do not defer requests for stopping sync to a thread pool. * Do not make pysqlcipher check if object is using in distinct threads. * Reset the sync loop when stopping the syncer. * Fix docstrings. * Check for _db_handle attribute when closing the database. --- client/src/leap/soledad/client/sqlcipher.py | 85 ++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 323d78f1..91821c25 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -46,6 +46,10 @@ import multiprocessing import os import threading import json +import u1db + +from u1db import errors as u1db_errors +from u1db.backends import sqlite_backend from hashlib import sha256 from contextlib import contextmanager @@ -53,10 +57,6 @@ from collections import defaultdict from httplib import CannotSendRequest from pysqlcipher import dbapi2 as sqlcipher_dbapi2 -from u1db.backends import sqlite_backend -from u1db import errors as u1db_errors -import u1db - from twisted.internet import reactor from twisted.internet.task import LoopingCall @@ -76,6 +76,7 @@ from leap.soledad.common.document import SoledadDocument logger = logging.getLogger(__name__) + # Monkey-patch u1db.backends.sqlite_backend with pysqlcipher.dbapi2 sqlite_backend.dbapi2 = sqlcipher_dbapi2 @@ -88,7 +89,7 @@ def initialize_sqlcipher_db(opts, on_init=None, check_same_thread=True): :type opts: SQLCipherOptions :param on_init: a tuple of queries to be executed on initialization :type on_init: tuple - :return: a SQLCipher connection + :return: pysqlcipher.dbapi2.Connection """ # Note: There seemed to be a bug in sqlite 3.5.9 (with python2.6) # where without re-opening the database on Windows, it @@ -104,6 +105,7 @@ def initialize_sqlcipher_db(opts, on_init=None, check_same_thread=True): set_init_pragmas(conn, opts, extra_queries=on_init) return conn + _db_init_lock = threading.Lock() @@ -146,6 +148,26 @@ class SQLCipherOptions(object): """ A container with options for the initialization of an SQLCipher database. """ + + @classmethod + def copy(cls, source, path=None, key=None, create=None, + is_raw_key=None, cipher=None, kdf_iter=None, cipher_page_size=None, + defer_encryption=None, sync_db_key=None): + """ + Return a copy of C{source} with parameters different than None + replaced by new values. + """ + return SQLCipherOptions( + path if path else source.path, + key if key else source.key, + create=create if create else source.create, + is_raw_key=is_raw_key if is_raw_key else source.is_raw_key, + cipher=cipher if cipher else source.cipher, + kdf_iter=kdf_iter if kdf_iter else source.kdf_iter, + cipher_page_size=cipher_page_size if cipher_page_size else source.cipher_page_size, + defer_encryption=defer_encryption if defer_encryption else source.defer_encryption, + sync_db_key=sync_db_key if sync_db_key else source.sync_db_key) + def __init__(self, path, key, create=True, is_raw_key=False, cipher='aes-256-cbc', kdf_iter=4000, cipher_page_size=1024, defer_encryption=False, sync_db_key=None): @@ -156,9 +178,6 @@ class SQLCipherOptions(object): True/False, should the database be created if it doesn't already exist? :param create: bool - :param crypto: An instance of SoledadCrypto so we can encrypt/decrypt - document contents when syncing. - :type crypto: soledad.crypto.SoledadCrypto :param is_raw_key: Whether ``password`` is a raw 64-char hex string or a passphrase that should be hashed to obtain the encyrption key. @@ -184,11 +203,25 @@ class SQLCipherOptions(object): self.defer_encryption = defer_encryption self.sync_db_key = sync_db_key + def __str__(self): + """ + Return string representation of options, for easy debugging. + + :return: String representation of options. + :rtype: str + """ + attr_names = filter(lambda a: not a.startswith('_'), dir(self)) + attr_str = [] + for a in attr_names: + attr_str.append(a + "=" + str(getattr(self, a))) + name = self.__class__.__name__ + return "%s(%s)" % (name, ', '.join(attr_str)) + + # # The SQLCipher database # - class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): """ A U1DB implementation that uses SQLCipher as its persistence layer. @@ -212,9 +245,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): *** IMPORTANT *** - :param soledad_crypto: - :type soldead_crypto: - :param opts: + :param opts: options for initialization of the SQLCipher database. :type opts: SQLCipherOptions """ # ensure the db is encrypted if the file already exists @@ -348,7 +379,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): logger.debug("SQLCipher backend: closing") # close the actual database - if self._db_handle is not None: + if getattr(self, '_db_handle', False): self._db_handle.close() self._db_handle = None @@ -445,8 +476,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): self._crypto = soledad_crypto self.__replica_uid = replica_uid - print "REPLICA UID (u1dbsync init)", replica_uid - self._sync_db_key = opts.sync_db_key self._sync_db = None self._sync_db_write_lock = None @@ -471,28 +500,28 @@ class SQLCipherU1DBSync(SQLCipherDatabase): self._reactor = reactor self._reactor.callWhenRunning(self._start) - self.ready = False + self.ready = True self._db_handle = None self._initialize_syncer_main_db() if defer_encryption: - self._initialize_sync_db() + self._initialize_sync_db(opts) # initialize syncing queue encryption pool self._sync_enc_pool = crypto.SyncEncrypterPool( self._crypto, self._sync_db, self._sync_db_write_lock) - # ------------------------------------------------------------------ + # ----------------------------------------------------------------- # From the documentation: If f returns a deferred, rescheduling # will not take place until the deferred has fired. The result # value is ignored. # TODO use this to avoid multiple sync attempts if the sync has not # finished! - # ------------------------------------------------------------------ + # ----------------------------------------------------------------- # XXX this was called sync_watcher --- trace any remnants - self._sync_loop = LoopingCall(self._encrypt_syncing_docs), + self._sync_loop = LoopingCall(self._encrypt_syncing_docs) self._sync_loop.start(self.ENCRYPT_LOOP_PERIOD) self.shutdownID = None @@ -522,7 +551,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): #import thread #print "initializing in thread", thread.get_ident() # --------------------------------------------------- - self._db_handle = initialize_sqlcipher_db( self._opts, check_same_thread=False) self._real_replica_uid = None @@ -557,10 +585,14 @@ class SQLCipherU1DBSync(SQLCipherDatabase): else: sync_db_path = ":memory:" - opts.path = sync_db_path - + # we copy incoming options because the opts object might be used + # somewhere else + sync_opts = SQLCipherOptions.copy( + opts, path=sync_db_path, create=True) self._sync_db = initialize_sqlcipher_db( - opts, on_init=self._sync_db_extra_init) + sync_opts, on_init=self._sync_db_extra_init, + check_same_thread=False) + pragmas.set_crypto_pragmas(self._sync_db, opts) # --------------------------------------------------------- @property @@ -615,7 +647,6 @@ class SQLCipherU1DBSync(SQLCipherDatabase): # callLater this same function after a timeout (deferLater) # Might want to keep track of retries and cancel too. # -------------------------------------------------------------- - print "Syncing to...", url kwargs = {'creds': creds, 'autocreate': autocreate, 'defer_decryption': defer_decryption} return self._defer_to_sync_threadpool(self._sync, url, **kwargs) @@ -629,6 +660,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase): # threadpool. log.msg("in _sync") + self.__url = url with self._syncer(url, creds=creds) as syncer: # XXX could mark the critical section here... try: @@ -652,7 +684,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase): """ Interrupt all ongoing syncs. """ - self._defer_to_sync_threadpool(self._stop_sync) + self._stop_sync() def _stop_sync(self): for url in self._syncers: @@ -769,6 +801,7 @@ class SQLCipherU1DBSync(SQLCipherDatabase): """ # stop the sync loop for deferred encryption if self._sync_loop is not None: + self._sync_loop.reset() self._sync_loop.stop() self._sync_loop = None # close all open syncers -- cgit v1.2.3 From c24452af4da078eaf15aa0841d8f8482886735f4 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 16:22:15 -0200 Subject: Several fixes in SoledadSyncTarget: * Fix arg passing to syncing failure method. * Do not try to start sync loop which should be already running. * Adapt to removal of old multiprocessing safe db, now accesses the sqlcipher database directly. --- client/src/leap/soledad/client/target.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/client/src/leap/soledad/client/target.py b/client/src/leap/soledad/client/target.py index ba61cdff..dd61c070 100644 --- a/client/src/leap/soledad/client/target.py +++ b/client/src/leap/soledad/client/target.py @@ -188,7 +188,7 @@ class DocumentSyncerThread(threading.Thread): self._doc_syncer.failure_callback( self._idx, self._total, self._exception) - self._failed_method(self) + self._failed_method() # we do not release the callback lock here because we # failed and so we don't want other threads to succeed. @@ -1296,7 +1296,6 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): # decrypt docs in case of deferred decryption if defer_decryption: - self._sync_loop.start() while self.clear_to_sync() is False: sleep(self.DECRYPT_LOOP_PERIOD) self._teardown_sync_loop() @@ -1362,11 +1361,11 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): encr = SyncEncrypterPool sql = ("SELECT content FROM %s WHERE doc_id=? and rev=?" % ( encr.TABLE_NAME,)) - res = self._sync_db.select(sql, (doc_id, doc_rev)) - try: - val = res.next() + res = self._fetchall(sql, (doc_id, doc_rev)) + if res: + val = res.pop() return val[0] - except StopIteration: + else: # no doc found return None @@ -1508,3 +1507,9 @@ class SoledadSyncTarget(HTTPSyncTarget, TokenBasedAuth): :type token: str """ TokenBasedAuth.set_token_credentials(self, uuid, token) + + def _fetchall(self, *args, **kwargs): + with self._sync_db: + c = self._sync_db.cursor() + c.execute(*args, **kwargs) + return c.fetchall() -- cgit v1.2.3 From d8c457680b79c202d54dcf9ea799a03b5ffc6c03 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 25 Nov 2014 16:25:33 -0200 Subject: Add local replica info to sync debug output. --- client/src/leap/soledad/client/sync.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index aa19ddab..1a5e2989 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -115,9 +115,10 @@ class SoledadSynchronizer(Synchronizer): " target generation: %d\n" " target trans id: %s\n" " target my gen: %d\n" - " target my trans_id: %s" + " target my trans_id: %s\n" + " source replica_uid: %s\n" % (self.target_replica_uid, target_gen, target_trans_id, - target_my_gen, target_my_trans_id)) + target_my_gen, target_my_trans_id, self.source._replica_uid)) # make sure we'll have access to target replica uid once it exists if self.target_replica_uid is None: -- cgit v1.2.3 From d0a0e92550bcc148fa236add5360ed581109ae6b Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 16 Dec 2014 15:33:46 -0200 Subject: Use Twisted trial for running tests. --- common/pkg/requirements-testing.pip | 1 + common/setup.cfg | 2 ++ common/setup.py | 2 +- .../src/leap/soledad/common/tests/u1db_tests/test_backends.py | 10 ++++++++++ .../src/leap/soledad/common/tests/u1db_tests/test_document.py | 3 +++ .../src/leap/soledad/common/tests/u1db_tests/test_http_app.py | 11 +++++++++++ .../leap/soledad/common/tests/u1db_tests/test_http_client.py | 4 ++++ .../soledad/common/tests/u1db_tests/test_http_database.py | 5 +++++ common/src/leap/soledad/common/tests/u1db_tests/test_https.py | 2 ++ common/src/leap/soledad/common/tests/u1db_tests/test_open.py | 2 ++ .../common/tests/u1db_tests/test_remote_sync_target.py | 5 +++++ .../soledad/common/tests/u1db_tests/test_sqlite_backend.py | 4 ++++ common/src/leap/soledad/common/tests/u1db_tests/test_sync.py | 6 ++++++ 13 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 common/setup.cfg diff --git a/common/pkg/requirements-testing.pip b/common/pkg/requirements-testing.pip index 9302450c..c72c9fc4 100644 --- a/common/pkg/requirements-testing.pip +++ b/common/pkg/requirements-testing.pip @@ -3,3 +3,4 @@ testscenarios leap.common leap.soledad.server leap.soledad.client +setuptools-trial diff --git a/common/setup.cfg b/common/setup.cfg new file mode 100644 index 00000000..c71bffa0 --- /dev/null +++ b/common/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test = trial diff --git a/common/setup.py b/common/setup.py index 3650a15a..92f4827d 100644 --- a/common/setup.py +++ b/common/setup.py @@ -267,7 +267,7 @@ setup( namespace_packages=["leap", "leap.soledad"], packages=find_packages('src', exclude=['leap.soledad.common.tests']), package_dir={'': 'src'}, - test_suite='leap.soledad.common.tests.load_tests', + test_suite='leap.soledad.common.tests', install_requires=utils.parse_requirements(), tests_require=utils.parse_requirements( reqfiles=['pkg/requirements-testing.pip']), diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py index 54adcde1..27fc50dc 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py @@ -40,6 +40,8 @@ from u1db.remote import ( http_database, ) +from unittest import skip + def make_http_database_for_test(test, replica_uid, path='test', *args): test.startServer() @@ -79,6 +81,7 @@ class TestAlternativeDocument(DocumentBase): """A (not very) alternative implementation of Document.""" +@skip("Skiping tests imported from U1DB.") class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): scenarios = tests.LOCAL_DATABASES_SCENARIOS + [ @@ -327,6 +330,7 @@ class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) +@skip("Skiping tests imported from U1DB.") class DocumentSizeTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -351,6 +355,7 @@ class DocumentSizeTests(tests.DatabaseBaseTests): self.assertEqual(1000000, self.db.document_size_limit) +@skip("Skiping tests imported from U1DB.") class LocalDatabaseTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -609,6 +614,7 @@ class LocalDatabaseTests(tests.DatabaseBaseTests): self.db.whats_changed(2)) +@skip("Skiping tests imported from U1DB.") class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -633,6 +639,7 @@ class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): self.db.validate_gen_and_trans_id, gen + 1, trans_id) +@skip("Skiping tests imported from U1DB.") class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -652,6 +659,7 @@ class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): self.db._validate_source, 'other', 1, 'T-sad') +@skip("Skiping tests imported from U1DB.") class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): # test supporting/functionality around storing conflicts @@ -1028,6 +1036,7 @@ class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): self.assertRaises(errors.ConflictedDoc, self.db.delete_doc, doc2) +@skip("Skiping tests imported from U1DB.") class DatabaseIndexTests(tests.DatabaseBaseTests): scenarios = tests.LOCAL_DATABASES_SCENARIOS @@ -1834,6 +1843,7 @@ class DatabaseIndexTests(tests.DatabaseBaseTests): self.assertParseError('combine(lower(x)x,foo)') +@skip("Skiping tests imported from U1DB.") class PythonBackendTests(tests.DatabaseBaseTests): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py index 8b30ed51..d8a27f51 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py @@ -15,11 +15,13 @@ # along with u1db. If not, see . +from unittest import skip from u1db import errors from leap.soledad.common.tests import u1db_tests as tests +@skip("Skiping tests imported from U1DB.") class TestDocument(tests.TestCase): scenarios = ([( @@ -83,6 +85,7 @@ class TestDocument(tests.TestCase): self.assertEqual(len('a' + 'b'), doc_a.get_size()) +@skip("Skiping tests imported from U1DB.") class TestPyDocument(tests.TestCase): scenarios = ([( diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py index 789006ba..522eb476 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_app.py @@ -24,6 +24,8 @@ except ImportError: import json # noqa import StringIO +from unittest import skip + from u1db import ( __version__ as _u1db_version, errors, @@ -38,6 +40,7 @@ from u1db.remote import ( ) +@skip("Skiping tests imported from U1DB.") class TestFencedReader(tests.TestCase): def test_init(self): @@ -145,6 +148,7 @@ class TestFencedReader(tests.TestCase): self.assertRaises(http_app.BadRequest, reader.getline) +@skip("Skiping tests imported from U1DB.") class TestHTTPMethodDecorator(tests.TestCase): def test_args(self): @@ -253,6 +257,7 @@ class parameters: max_entry_size = 100000 +@skip("Skiping tests imported from U1DB.") class TestHTTPInvocationByMethodWithBody(tests.TestCase): def test_get(self): @@ -433,6 +438,7 @@ class TestHTTPInvocationByMethodWithBody(tests.TestCase): self.assertRaises(http_app.BadRequest, invoke) +@skip("Skiping tests imported from U1DB.") class TestHTTPResponder(tests.TestCase): def start_response(self, status, headers): @@ -521,6 +527,7 @@ class TestHTTPResponder(tests.TestCase): responder.content) +@skip("Skiping tests imported from U1DB.") class TestHTTPApp(tests.TestCase): def setUp(self): @@ -949,6 +956,7 @@ class TestHTTPApp(tests.TestCase): self.assertEqual({'error': 'unavailable'}, json.loads(parts[2])) +@skip("Skiping tests imported from U1DB.") class TestRequestHooks(tests.TestCase): def setUp(self): @@ -1000,12 +1008,14 @@ class TestRequestHooks(tests.TestCase): self.assertEqual(['begin', 'bad-request'], calls) +@skip("Skiping tests imported from U1DB.") class TestHTTPErrors(tests.TestCase): def test_wire_description_to_status(self): self.assertNotIn("error", http_errors.wire_description_to_status) +@skip("Skiping tests imported from U1DB.") class TestHTTPAppErrorHandling(tests.TestCase): def setUp(self): @@ -1113,6 +1123,7 @@ class TestHTTPAppErrorHandling(tests.TestCase): self.assertEqual(self.exc, exc) +@skip("Skiping tests imported from U1DB.") class TestPluggableSyncExchange(tests.TestCase): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py index 08e9714e..f9e09cbd 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py @@ -26,6 +26,8 @@ from u1db import ( errors, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( @@ -33,6 +35,7 @@ from u1db.remote import ( ) +@skip("Skiping tests imported from U1DB.") class TestEncoder(tests.TestCase): def test_encode_string(self): @@ -45,6 +48,7 @@ class TestEncoder(tests.TestCase): self.assertEqual("false", http_client._encode_query_parameter(False)) +@skip("Skiping tests imported from U1DB.") class TestHTTPClientBase(tests.TestCaseWithServer): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py index 9251000e..bf7ed5d3 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py @@ -27,6 +27,8 @@ from u1db import ( Document, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( @@ -38,6 +40,7 @@ from leap.soledad.common.tests.u1db_tests.test_remote_sync_target import ( ) +@skip("Skiping tests imported from U1DB.") class TestHTTPDatabaseSimpleOperations(tests.TestCase): def setUp(self): @@ -190,6 +193,7 @@ class TestHTTPDatabaseSimpleOperations(tests.TestCase): self.assertEqual(self.db._creds, st._creds) +@skip("Skiping tests imported from U1DB.") class TestHTTPDatabaseCtrWithCreds(tests.TestCase): def test_ctr_with_creds(self): @@ -202,6 +206,7 @@ class TestHTTPDatabaseCtrWithCreds(tests.TestCase): self.assertIn('oauth', db1._creds) +@skip("Skiping tests imported from U1DB.") class TestHTTPDatabaseIntegration(tests.TestCaseWithServer): make_app_with_state = staticmethod(make_http_app) diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py index c5b316ed..cea175d6 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py @@ -5,6 +5,7 @@ import ssl import sys from paste import httpserver +from unittest import skip from u1db.remote import ( http_client, @@ -51,6 +52,7 @@ def oauth_https_sync_target(test, host, path): return st +@skip("Skiping tests imported from U1DB.") class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): scenarios = [ diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py index 63406245..ee249e6e 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py @@ -22,12 +22,14 @@ from u1db import ( errors, open as u1db_open, ) +from unittest import skip from leap.soledad.common.tests import u1db_tests as tests from u1db.backends import sqlite_backend from leap.soledad.common.tests.u1db_tests.test_backends \ import TestAlternativeDocument +@skip("Skiping tests imported from U1DB.") class TestU1DBOpen(tests.TestCase): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py b/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py index 3793e0df..bd7e4103 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_remote_sync_target.py @@ -22,6 +22,8 @@ from u1db import ( errors, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.remote import ( @@ -31,6 +33,7 @@ from u1db.remote import ( ) +@skip("Skiping tests imported from U1DB.") class TestHTTPSyncTargetBasics(tests.TestCase): def test_parse_url(self): @@ -41,6 +44,7 @@ class TestHTTPSyncTargetBasics(tests.TestCase): self.assertEqual('/', remote_target._url.path) +@skip("Skiping tests imported from U1DB.") class TestParsingSyncStream(tests.TestCase): def test_wrong_start(self): @@ -130,6 +134,7 @@ def oauth_http_sync_target(test, path): return st +@skip("Skiping tests imported from U1DB.") class TestRemoteSyncTargets(tests.TestCaseWithServer): scenarios = [ diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py index 8292dd07..aed8a6e5 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sqlite_backend.py @@ -27,6 +27,8 @@ from u1db import ( query_parser, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.backends import sqlite_backend @@ -38,6 +40,7 @@ simple_doc = '{"key": "value"}' nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}' +@skip("Skiping tests imported from U1DB.") class TestSQLiteDatabase(tests.TestCase): def test_atomic_initialize(self): @@ -83,6 +86,7 @@ class TestSQLiteDatabase(tests.TestCase): self.assertTrue(db2._is_initialized(db1._get_sqlite_handle().cursor())) +@skip("Skiping tests imported from U1DB.") class TestSQLitePartialExpandDatabase(tests.TestCase): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py index 5e2bec86..bac1f177 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_sync.py @@ -26,6 +26,8 @@ from u1db import ( SyncTarget, ) +from unittest import skip + from leap.soledad.common.tests import u1db_tests as tests from u1db.backends import ( @@ -74,6 +76,7 @@ target_scenarios = [ ] +@skip("Skiping tests imported from U1DB.") class DatabaseSyncTargetTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): @@ -462,6 +465,7 @@ sync_scenarios.append(('pyhttp', { })) +@skip("Skiping tests imported from U1DB.") class DatabaseSyncTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): @@ -1118,6 +1122,7 @@ class DatabaseSyncTests(tests.DatabaseBaseTests, errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) +@skip("Skiping tests imported from U1DB.") class TestDbSync(tests.TestCaseWithServer): """Test db.sync remote sync shortcut""" @@ -1190,6 +1195,7 @@ class TestDbSync(tests.TestCaseWithServer): self.assertEqual(1, s_gen) +@skip("Skiping tests imported from U1DB.") class TestRemoteSyncIntegration(tests.TestCaseWithServer): """Integration tests for the most common sync scenario local -> remote""" -- cgit v1.2.3 From 0b88ef70ec12d3666a9bfc32481d672cb01cf056 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 17 Dec 2014 14:51:55 -0200 Subject: Do not try to close db syncer if db is not syncable. --- client/src/leap/soledad/client/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 998e9148..81bf1fd9 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -285,7 +285,8 @@ class Soledad(object): """ logger.debug("Closing soledad") self._dbpool.close() - self._dbsyncer.close() + if getattr(self, '_dbsyncer', None): + self._dbsyncer.close() # # ILocalStorage -- cgit v1.2.3 From 9f0e5ac8db4813b1277c3a858cf1d5cb785a4023 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 17 Dec 2014 14:53:08 -0200 Subject: Do not try to unlock shared db if db is not syncable. --- client/src/leap/soledad/client/shared_db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index 26ddc285..f1a2642e 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -178,5 +178,6 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): :raise HTTPError: """ - res, headers = self._request_json('DELETE', ['lock', self._uuid], - params={'token': token}) + if self.syncable: + _, _ = self._request_json( + 'DELETE', ['lock', self._uuid], params={'token': token}) -- cgit v1.2.3 From c654d5e777c0d1db75b5f3586bd20ce2ec4edadc Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 5 Jan 2015 10:46:31 -0400 Subject: add raw sqlcipher query method --- client/src/leap/soledad/client/adbapi.py | 1 + client/src/leap/soledad/client/api.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index 9ae2889e..f0b7f182 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -106,6 +106,7 @@ class U1DBConnection(adbapi.Connection): else: return getattr(self._connection, name) + class U1DBTransaction(adbapi.Transaction): """ A wrapper for a U1DB 'cursor' object. diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 81bf1fd9..88bb4969 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -778,6 +778,16 @@ class Soledad(object): """ self._secrets.change_passphrase(new_passphrase) + # + # Raw SQLCIPHER Queries + # + + def raw_sqlcipher_query(self, *args, **kw): + """ + Run a raw sqlcipher query in the local database. + """ + return self._dbpool.runQuery(*args, **kw) + def _convert_to_unicode(content): """ -- cgit v1.2.3 From 661ee9eb1ce29056f219a2a95688840cebcf889a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 23 Jan 2015 14:54:08 -0400 Subject: Bail out ddocs installation if the path doesn't exist. Fix: #6671 --- common/MANIFEST.in | 3 +++ common/changes/bug_6671-bail-out-if-no-cdocs-dir | 1 + common/setup.py | 5 +++++ 3 files changed, 9 insertions(+) create mode 100644 common/changes/bug_6671-bail-out-if-no-cdocs-dir diff --git a/common/MANIFEST.in b/common/MANIFEST.in index 8e5a2342..a26a12a6 100644 --- a/common/MANIFEST.in +++ b/common/MANIFEST.in @@ -2,4 +2,7 @@ include pkg/* include versioneer.py include LICENSE include CHANGELOG + +# What do we want the ddocs folder in the source package for? -- kali +# it should be enough with having the compiled stuff. recursive-include src/leap/soledad/common/ddocs * diff --git a/common/changes/bug_6671-bail-out-if-no-cdocs-dir b/common/changes/bug_6671-bail-out-if-no-cdocs-dir new file mode 100644 index 00000000..e57e50e5 --- /dev/null +++ b/common/changes/bug_6671-bail-out-if-no-cdocs-dir @@ -0,0 +1 @@ +o Bail out if cdocs/ dir does not exist. Closes: #6671 diff --git a/common/setup.py b/common/setup.py index 92f4827d..b0ab8352 100644 --- a/common/setup.py +++ b/common/setup.py @@ -155,6 +155,11 @@ def build_ddocs_py(basedir=None, with_src=True): dest_prefix = join(basedir, *dest_common_path) ddocs_prefix = join(prefix, 'ddocs') + + if not isdir(ddocs_prefix): + print "No ddocs/ folder, bailing out..." + return + ddocs = {} # design docs are represented by subdirectories of `ddocs_prefix` -- cgit v1.2.3 From 61212438a57d2450db767860c6e09e43d9e53532 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 23 Dec 2014 02:10:47 -0400 Subject: add some benchmarking skeleton --- .../soledad/client/examples/benchmarks/.gitignore | 1 + .../client/examples/benchmarks/get_sample.sh | 3 + .../examples/benchmarks/measure_index_times.py | 177 +++++++++++++++++++++ .../benchmarks/measure_index_times_custom_docid.py | 177 +++++++++++++++++++++ 4 files changed, 358 insertions(+) create mode 100644 client/src/leap/soledad/client/examples/benchmarks/.gitignore create mode 100755 client/src/leap/soledad/client/examples/benchmarks/get_sample.sh create mode 100644 client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py create mode 100644 client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py diff --git a/client/src/leap/soledad/client/examples/benchmarks/.gitignore b/client/src/leap/soledad/client/examples/benchmarks/.gitignore new file mode 100644 index 00000000..2211df63 --- /dev/null +++ b/client/src/leap/soledad/client/examples/benchmarks/.gitignore @@ -0,0 +1 @@ +*.txt diff --git a/client/src/leap/soledad/client/examples/benchmarks/get_sample.sh b/client/src/leap/soledad/client/examples/benchmarks/get_sample.sh new file mode 100755 index 00000000..1995eee1 --- /dev/null +++ b/client/src/leap/soledad/client/examples/benchmarks/get_sample.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mkdir tmp +wget http://www.gutenberg.org/cache/epub/101/pg101.txt -O hacker_crackdown.txt diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py new file mode 100644 index 00000000..7fa1e38f --- /dev/null +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# measure_index_times.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Measure u1db retrieval times for different u1db index situations. +""" +from __future__ import print_function +from functools import partial +import datetime +import hashlib +import os +import sys + +import u1db +from twisted.internet import defer, reactor + +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +folder = os.environ.get("TMPDIR", "tmp") +numdocs = int(os.environ.get("DOCS", "1000")) +silent = os.environ.get("SILENT", False) +tmpdb = os.path.join(folder, "test.soledad") + + +sample_file = os.environ.get("SAMPLE", "hacker_crackdown.txt") +sample_path = os.path.join(os.curdir, sample_file) + +try: + with open(sample_file) as f: + SAMPLE = f.readlines() +except Exception: + print("[!] Problem opening sample file. Did you download " + "the sample, or correctly set 'SAMPLE' env var?") + sys.exit(1) + +if numdocs > len(SAMPLE): + print("[!] Sorry! The requested DOCS number is larger than " + "the num of lines in our sample file") + sys.exit(1) + + +def debug(*args): + if not silent: + print(*args) + +debug("[+] db path:", tmpdb) +debug("[+] num docs", numdocs) + +if os.path.isfile(tmpdb): + debug("[+] Removing existing db file...") + os.remove(tmpdb) + +start_time = datetime.datetime.now() + +opts = SQLCipherOptions(tmpdb, "secret", create=True) +dbpool = adbapi.getConnectionPool(opts) + + +def createDoc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + +db_indexes = { + 'by-chash': ['chash'], + 'by-number': ['number']} + + +def create_indexes(_): + deferreds = [] + for index, definition in db_indexes.items(): + d = dbpool.runU1DBQuery("create_index", index, *definition) + deferreds.append(d) + return defer.gatherResults(deferreds) + + +class TimeWitness(object): + def __init__(self, init_time): + self.init_time = init_time + + def get_time_count(self): + return datetime.datetime.now() - self.init_time + + +def get_from_index(_): + init_time = datetime.datetime.now() + debug("GETTING FROM INDEX...", init_time) + + def printValue(res, time): + print("RESULT->", res) + print("Index Query Took: ", time.get_time_count()) + return res + + d = dbpool.runU1DBQuery( + "get_from_index", "by-chash", + #"1150c7f10fabce0a57ce13071349fc5064f15bdb0cc1bf2852f74ef3f103aff5") + # XXX this is line 89 from the hacker crackdown... + # Should accept any other optional hash as an enviroment variable. + "57793320d4997a673fc7062652da0596c36a4e9fbe31310d2281e67d56d82469") + d.addCallback(printValue, TimeWitness(init_time)) + return d + + +def getAllDocs(): + return dbpool.runU1DBQuery("get_all_docs") + + +def errBack(e): + debug("[!] ERROR FOUND!!!") + e.printTraceback() + reactor.stop() + + +def countDocs(_): + debug("counting docs...") + d = getAllDocs() + d.addCallbacks(printResult, errBack) + d.addCallbacks(allDone, errBack) + return d + + +def printResult(r, **kwargs): + if kwargs: + debug(*kwargs.values()) + elif isinstance(r, u1db.Document): + debug(r.doc_id, r.content['number']) + else: + len_results = len(r[1]) + debug("GOT %s results" % len(r[1])) + + if len_results == numdocs: + debug("ALL GOOD") + else: + debug("[!] MISSING DOCS!!!!!") + raise ValueError("We didn't expect this result len") + + +def allDone(_): + debug("ALL DONE!") + + #if silent: + end_time = datetime.datetime.now() + print((end_time - start_time).total_seconds()) + reactor.stop() + + +def insert_docs(_): + deferreds = [] + for i in range(numdocs): + payload = SAMPLE[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = createDoc(doc) + d.addCallbacks(partial(printResult, i=i, chash=chash, payload=payload), + lambda e: e.printTraceback()) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + +d = create_indexes(None) +d.addCallback(insert_docs) +d.addCallback(get_from_index) +d.addCallback(countDocs) + +reactor.run() diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py new file mode 100644 index 00000000..c6d76e6b --- /dev/null +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py @@ -0,0 +1,177 @@ +# -*- coding: utf-8 -*- +# measure_index_times.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Measure u1db retrieval times for different u1db index situations. +""" +from __future__ import print_function +from functools import partial +import datetime +import hashlib +import os +import sys + +import u1db +from twisted.internet import defer, reactor + +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +folder = os.environ.get("TMPDIR", "tmp") +numdocs = int(os.environ.get("DOCS", "1000")) +silent = os.environ.get("SILENT", False) +tmpdb = os.path.join(folder, "test.soledad") + + +sample_file = os.environ.get("SAMPLE", "hacker_crackdown.txt") +sample_path = os.path.join(os.curdir, sample_file) + +try: + with open(sample_file) as f: + SAMPLE = f.readlines() +except Exception: + print("[!] Problem opening sample file. Did you download " + "the sample, or correctly set 'SAMPLE' env var?") + sys.exit(1) + +if numdocs > len(SAMPLE): + print("[!] Sorry! The requested DOCS number is larger than " + "the num of lines in our sample file") + sys.exit(1) + + +def debug(*args): + if not silent: + print(*args) + +debug("[+] db path:", tmpdb) +debug("[+] num docs", numdocs) + +if os.path.isfile(tmpdb): + debug("[+] Removing existing db file...") + os.remove(tmpdb) + +start_time = datetime.datetime.now() + +opts = SQLCipherOptions(tmpdb, "secret", create=True) +dbpool = adbapi.getConnectionPool(opts) + + +def createDoc(doc, doc_id): + return dbpool.runU1DBQuery("create_doc", doc, doc_id=doc_id) + +db_indexes = { + 'by-chash': ['chash'], + 'by-number': ['number']} + + +def create_indexes(_): + deferreds = [] + for index, definition in db_indexes.items(): + d = dbpool.runU1DBQuery("create_index", index, *definition) + deferreds.append(d) + return defer.gatherResults(deferreds) + + +class TimeWitness(object): + def __init__(self, init_time): + self.init_time = init_time + + def get_time_count(self): + return datetime.datetime.now() - self.init_time + + +def get_from_index(_): + init_time = datetime.datetime.now() + debug("GETTING FROM INDEX...", init_time) + + def printValue(res, time): + print("RESULT->", res) + print("Index Query Took: ", time.get_time_count()) + return res + + d = dbpool.runU1DBQuery( + "get_doc", + #"1150c7f10fabce0a57ce13071349fc5064f15bdb0cc1bf2852f74ef3f103aff5") + # XXX this is line 89 from the hacker crackdown... + # Should accept any other optional hash as an enviroment variable. + "57793320d4997a673fc7062652da0596c36a4e9fbe31310d2281e67d56d82469") + d.addCallback(printValue, TimeWitness(init_time)) + return d + + +def getAllDocs(): + return dbpool.runU1DBQuery("get_all_docs") + + +def errBack(e): + debug("[!] ERROR FOUND!!!") + e.printTraceback() + reactor.stop() + + +def countDocs(_): + debug("counting docs...") + d = getAllDocs() + d.addCallbacks(printResult, errBack) + d.addCallbacks(allDone, errBack) + return d + + +def printResult(r, **kwargs): + if kwargs: + debug(*kwargs.values()) + elif isinstance(r, u1db.Document): + debug(r.doc_id, r.content['number']) + else: + len_results = len(r[1]) + debug("GOT %s results" % len(r[1])) + + if len_results == numdocs: + debug("ALL GOOD") + else: + debug("[!] MISSING DOCS!!!!!") + raise ValueError("We didn't expect this result len") + + +def allDone(_): + debug("ALL DONE!") + + #if silent: + end_time = datetime.datetime.now() + print((end_time - start_time).total_seconds()) + reactor.stop() + + +def insert_docs(_): + deferreds = [] + for i in range(numdocs): + payload = SAMPLE[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = createDoc(doc, doc_id=chash) + d.addCallbacks(partial(printResult, i=i, chash=chash, payload=payload), + lambda e: e.printTraceback()) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + +d = create_indexes(None) +d.addCallback(insert_docs) +d.addCallback(get_from_index) +d.addCallback(countDocs) + +reactor.run() -- cgit v1.2.3 From 14f34b1f64a667bf4a146e8579f95c5d308a1f77 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 8 Jan 2015 18:57:38 -0200 Subject: Retry on SQLCipher timeout (#6625). --- .../bug_6625_retry-on-sqlcipher-thread-timeout | 1 + client/src/leap/soledad/client/adbapi.py | 45 +- .../leap/soledad/common/tests/hacker_crackdown.txt | 13005 +++++++++++++++++++ common/src/leap/soledad/common/tests/test_async.py | 144 + 4 files changed, 13193 insertions(+), 2 deletions(-) create mode 100644 client/changes/bug_6625_retry-on-sqlcipher-thread-timeout create mode 100644 common/src/leap/soledad/common/tests/hacker_crackdown.txt create mode 100644 common/src/leap/soledad/common/tests/test_async.py diff --git a/client/changes/bug_6625_retry-on-sqlcipher-thread-timeout b/client/changes/bug_6625_retry-on-sqlcipher-thread-timeout new file mode 100644 index 00000000..8b2ce055 --- /dev/null +++ b/client/changes/bug_6625_retry-on-sqlcipher-thread-timeout @@ -0,0 +1 @@ + o Retry on sqlcipher thread timeouts (#6625). diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index f0b7f182..7ad10db5 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -21,20 +21,37 @@ It uses twisted.enterprise.adbapi. import re import os import sys +import logging from functools import partial +from threading import BoundedSemaphore from twisted.enterprise import adbapi from twisted.python import log from zope.proxy import ProxyBase, setProxiedObject +from pysqlcipher.dbapi2 import OperationalError from leap.soledad.client import sqlcipher as soledad_sqlcipher +logger = logging.getLogger(name=__name__) + + DEBUG_SQL = os.environ.get("LEAP_DEBUG_SQL") if DEBUG_SQL: log.startLogging(sys.stdout) +""" +How long the SQLCipher connection should wait for the lock to go away until +raising an exception. +""" +SQLCIPHER_CONNECTION_TIMEOUT = 10 + +""" +How many times a SQLCipher query should be retried in case of timeout. +""" +SQLCIPHER_MAX_RETRIES = 10 + def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): """ @@ -58,7 +75,8 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher"): openfun = partial(soledad_sqlcipher.set_init_pragmas, opts=opts) return U1DBConnectionPool( "%s.dbapi2" % driver, database=opts.path, - check_same_thread=False, cp_openfun=openfun) + check_same_thread=False, cp_openfun=openfun, + timeout=SQLCIPHER_CONNECTION_TIMEOUT) class U1DBConnection(adbapi.Connection): @@ -154,6 +172,10 @@ class U1DBConnectionPool(adbapi.ConnectionPool): """ Execute a U1DB query in a thread, using a pooled connection. + Concurrent threads trying to update the same database may timeout + because of other threads holding the database lock. Because of this, + we will retry SQLCIPHER_MAX_RETRIES times and fail after that. + :param meth: The U1DB wrapper method name. :type meth: str @@ -162,7 +184,26 @@ class U1DBConnectionPool(adbapi.ConnectionPool): :rtype: twisted.internet.defer.Deferred """ meth = "u1db_%s" % meth - return self.runInteraction(self._runU1DBQuery, meth, *args, **kw) + semaphore = BoundedSemaphore(SQLCIPHER_MAX_RETRIES - 1) + + def _run_interaction(): + return self.runInteraction( + self._runU1DBQuery, meth, *args, **kw) + + def _errback(failure): + failure.trap(OperationalError) + if failure.getErrorMessage() == "database is locked": + should_retry = semaphore.acquire(False) + if should_retry: + logger.warning( + "Database operation timed out while waiting for " + "lock, trying again...") + return _run_interaction() + return failure + + d = _run_interaction() + d.addErrback(_errback) + return d def _runU1DBQuery(self, trans, meth, *args, **kw): """ diff --git a/common/src/leap/soledad/common/tests/hacker_crackdown.txt b/common/src/leap/soledad/common/tests/hacker_crackdown.txt new file mode 100644 index 00000000..a01eb509 --- /dev/null +++ b/common/src/leap/soledad/common/tests/hacker_crackdown.txt @@ -0,0 +1,13005 @@ +The Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + +** This is a COPYRIGHTED Project Gutenberg eBook, Details Below ** +** Please follow the copyright guidelines in this file. ** + +Title: Hacker Crackdown + Law and Disorder on the Electronic Frontier + +Author: Bruce Sterling + +Posting Date: February 9, 2012 [EBook #101] +Release Date: January, 1994 + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** + + + + + + + + + + + + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + +CONTENTS + + +Preface to the Electronic Release of The Hacker Crackdown + +Chronology of the Hacker Crackdown + + +Introduction + + +Part 1: CRASHING THE SYSTEM +A Brief History of Telephony +Bell's Golden Vaporware +Universal Service +Wild Boys and Wire Women +The Electronic Communities +The Ungentle Giant +The Breakup +In Defense of the System +The Crash Post-Mortem +Landslides in Cyberspace + + +Part 2: THE DIGITAL UNDERGROUND +Steal This Phone +Phreaking and Hacking +The View From Under the Floorboards +Boards: Core of the Underground +Phile Phun +The Rake's Progress +Strongholds of the Elite +Sting Boards +Hot Potatoes +War on the Legion +Terminus +Phile 9-1-1 +War Games +Real Cyberpunk + + +Part 3: LAW AND ORDER +Crooked Boards +The World's Biggest Hacker Bust +Teach Them a Lesson +The U.S. Secret Service +The Secret Service Battles the Boodlers +A Walk Downtown +FCIC: The Cutting-Edge Mess +Cyberspace Rangers +FLETC: Training the Hacker-Trackers + + +Part 4: THE CIVIL LIBERTARIANS +NuPrometheus + FBI = Grateful Dead +Whole Earth + Computer Revolution = WELL +Phiber Runs Underground and Acid Spikes the Well +The Trial of Knight Lightning +Shadowhawk Plummets to Earth +Kyrie in the Confessional +$79,499 +A Scholar Investigates +Computers, Freedom, and Privacy + + +Electronic Afterword to The Hacker Crackdown, Halloween 1993 + + + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + + +Preface to the Electronic Release of The Hacker Crackdown + + +January 1, 1994--Austin, Texas + + +Hi, I'm Bruce Sterling, the author of this electronic book. + +Out in the traditional world of print, The Hacker Crackdown +is ISBN 0-553-08058-X, and is formally catalogued by +the Library of Congress as "1. Computer crimes--United States. +2. Telephone--United States--Corrupt practices. +3. Programming (Electronic computers)--United States--Corrupt practices." + +`Corrupt practices,' I always get a kick out of that description. +Librarians are very ingenious people. + +The paperback is ISBN 0-553-56370-X. If you go +and buy a print version of The Hacker Crackdown, +an action I encourage heartily, you may notice that +in the front of the book, beneath the copyright notice-- +"Copyright (C) 1992 by Bruce Sterling"-- +it has this little block of printed legal +boilerplate from the publisher. It says, and I quote: + + "No part of this book may be reproduced or transmitted in any form +or by any means, electronic or mechanical, including photocopying, +recording, or by any information storage and retrieval system, +without permission in writing from the publisher. +For information address: Bantam Books." + +This is a pretty good disclaimer, as such disclaimers go. +I collect intellectual-property disclaimers, and I've seen dozens of them, +and this one is at least pretty straightforward. In this narrow +and particular case, however, it isn't quite accurate. +Bantam Books puts that disclaimer on every book they publish, +but Bantam Books does not, in fact, own the electronic rights to this book. +I do, because of certain extensive contract maneuverings my agent and I +went through before this book was written. I want to give those electronic +publishing rights away through certain not-for-profit channels, +and I've convinced Bantam that this is a good idea. + +Since Bantam has seen fit to peacably agree to this scheme of mine, +Bantam Books is not going to fuss about this. Provided you don't try +to sell the book, they are not going to bother you for what you do with +the electronic copy of this book. If you want to check this out personally, +you can ask them; they're at 1540 Broadway NY NY 10036. However, if you were +so foolish as to print this book and start retailing it for money in violation +of my copyright and the commercial interests of Bantam Books, then Bantam, +a part of the gigantic Bertelsmann multinational publishing combine, +would roust some of their heavy-duty attorneys out of hibernation +and crush you like a bug. This is only to be expected. +I didn't write this book so that you could make money out of it. +If anybody is gonna make money out of this book, +it's gonna be me and my publisher. + +My publisher deserves to make money out of this book. +Not only did the folks at Bantam Books commission me +to write the book, and pay me a hefty sum to do so, but +they bravely printed, in text, an electronic document the +reproduction of which was once alleged to be a federal felony. +Bantam Books and their numerous attorneys were very brave +and forthright about this book. Furthermore, my former editor +at Bantam Books, Betsy Mitchell, genuinely cared about this project, +and worked hard on it, and had a lot of wise things to say +about the manuscript. Betsy deserves genuine credit for this book, +credit that editors too rarely get. + +The critics were very kind to The Hacker Crackdown, +and commercially the book has done well. On the other hand, +I didn't write this book in order to squeeze every last nickel +and dime out of the mitts of impoverished sixteen-year-old +cyberpunk high-school-students. Teenagers don't have any money-- +(no, not even enough for the six-dollar Hacker Crackdown paperback, +with its attractive bright-red cover and useful index). +That's a major reason why teenagers sometimes succumb to the temptation +to do things they shouldn't, such as swiping my books out of libraries. +Kids: this one is all yours, all right? Go give the print version back. +*8-) + +Well-meaning, public-spirited civil libertarians don't have much money, +either. And it seems almost criminal to snatch cash out of the hands of +America's direly underpaid electronic law enforcement community. + +If you're a computer cop, a hacker, or an electronic civil +liberties activist, you are the target audience for this book. +I wrote this book because I wanted to help you, and help other people +understand you and your unique, uhm, problems. I wrote this book +to aid your activities, and to contribute to the public discussion +of important political issues. In giving the text away in this +fashion, I am directly contributing to the book's ultimate aim: +to help civilize cyberspace. + +Information WANTS to be free. And the information inside +this book longs for freedom with a peculiar intensity. +I genuinely believe that the natural habitat of this book +is inside an electronic network. That may not be the easiest +direct method to generate revenue for the book's author, +but that doesn't matter; this is where this book belongs +by its nature. I've written other books--plenty of other books-- +and I'll write more and I am writing more, but this one is special. +I am making The Hacker Crackdown available electronically +as widely as I can conveniently manage, and if you like the book, +and think it is useful, then I urge you to do the same with it. + +You can copy this electronic book. Copy the heck out of it, +be my guest, and give those copies to anybody who wants them. +The nascent world of cyberspace is full of sysadmins, teachers, +trainers, cybrarians, netgurus, and various species of cybernetic activist. +If you're one of those people, I know about you, and I know the hassle +you go through to try to help people learn about the electronic frontier. +I hope that possessing this book in electronic form will lessen your troubles. +Granted, this treatment of our electronic social spectrum is not the ultimate +in academic rigor. And politically, it has something to offend +and trouble almost everyone. But hey, I'm told it's readable, +and at least the price is right. + +You can upload the book onto bulletin board systems, or Internet nodes, +or electronic discussion groups. Go right ahead and do that, I am giving +you express permission right now. Enjoy yourself. + +You can put the book on disks and give the disks away, +as long as you don't take any money for it. + +But this book is not public domain. You can't copyright it in +your own name. I own the copyright. Attempts to pirate this book +and make money from selling it may involve you in a serious litigative snarl. +Believe me, for the pittance you might wring out of such an action, +it's really not worth it. This book don't "belong" to you. +In an odd but very genuine way, I feel it doesn't "belong" to me, either. +It's a book about the people of cyberspace, and distributing it in this way +is the best way I know to actually make this information available, +freely and easily, to all the people of cyberspace--including people +far outside the borders of the United States, who otherwise may never +have a chance to see any edition of the book, and who may perhaps learn +something useful from this strange story of distant, obscure, but portentous +events in so-called "American cyberspace." + +This electronic book is now literary freeware. It now belongs to the +emergent realm of alternative information economics. You have no right +to make this electronic book part of the conventional flow of commerce. +Let it be part of the flow of knowledge: there's a difference. +I've divided the book into four sections, so that it is less ungainly +for upload and download; if there's a section of particular relevance +to you and your colleagues, feel free to reproduce that one and skip the rest. + +[Project Gutenberg has reassembled the file, with Sterling's permission.] + +Just make more when you need them, and give them to whoever might want them. + +Now have fun. + +Bruce Sterling--bruces@well.sf.ca.us + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + + + + +CHRONOLOGY OF THE HACKER CRACKDOWN + + +1865 U.S. Secret Service (USSS) founded. + +1876 Alexander Graham Bell invents telephone. + +1878 First teenage males flung off phone system by enraged authorities. + +1939 "Futurian" science-fiction group raided by Secret Service. + +1971 Yippie phone phreaks start YIPL/TAP magazine. + +1972 RAMPARTS magazine seized in blue-box rip-off scandal. + +1978 Ward Christenson and Randy Suess create first personal + computer bulletin board system. + +1982 William Gibson coins term "cyberspace." + +1982 "414 Gang" raided. + +1983-1983 AT&T dismantled in divestiture. + +1984 Congress passes Comprehensive Crime Control Act giving USSS + jurisdiction over credit card fraud and computer fraud. + +1984 "Legion of Doom" formed. + +1984. 2600: THE HACKER QUARTERLY founded. + +1984. WHOLE EARTH SOFTWARE CATALOG published. + +1985. First police "sting" bulletin board systems established. + +1985. Whole Earth 'Lectronic Link computer conference (WELL) goes on-line. + +1986 Computer Fraud and Abuse Act passed. + +1986 Electronic Communications Privacy Act passed. + +1987 Chicago prosecutors form Computer Fraud and Abuse Task Force. + + +1988 + +July. Secret Service covertly videotapes "SummerCon" hacker convention. + +September. "Prophet" cracks BellSouth AIMSX computer network + and downloads E911 Document to his own computer and to Jolnet. + +September. AT&T Corporate Information Security informed of Prophet's action. + +October. Bellcore Security informed of Prophet's action. + + +1989 + +January. Prophet uploads E911 Document to Knight Lightning. + +February 25. Knight Lightning publishes E911 Document in PHRACK + electronic newsletter. + +May. Chicago Task Force raids and arrests "Kyrie." + +June. "NuPrometheus League" distributes Apple Computer proprietary software. + +June 13. Florida probation office crossed with phone-sex line + in switching-station stunt. + +July. "Fry Guy" raided by USSS and Chicago Computer Fraud + and Abuse Task Force. + +July. Secret Service raids "Prophet," "Leftist," and "Urvile" in Georgia. + + +1990 + +January 15. Martin Luther King Day Crash strikes AT&T long-distance + network nationwide. + +January 18-19. Chicago Task Force raids Knight Lightning in St. Louis. + +January 24. USSS and New York State Police raid "Phiber Optik," + "Acid Phreak," and "Scorpion" in New York City. + +February 1. USSS raids "Terminus" in Maryland. + +February 3. Chicago Task Force raids Richard Andrews' home. + +February 6. Chicago Task Force raids Richard Andrews' business. + +February 6. USSS arrests Terminus, Prophet, Leftist, and Urvile. + +February 9. Chicago Task Force arrests Knight Lightning. + +February 20. AT&T Security shuts down public-access + "attctc" computer in Dallas. + +February 21. Chicago Task Force raids Robert Izenberg in Austin. + +March 1. Chicago Task Force raids Steve Jackson Games, Inc., + "Mentor," and "Erik Bloodaxe" in Austin. + +May 7,8,9. + +USSS and Arizona Organized Crime and Racketeering Bureau conduct +"Operation Sundevil" raids in Cincinnatti, Detroit, Los Angeles, +Miami, Newark, Phoenix, Pittsburgh, Richmond, Tucson, San Diego, +San Jose, and San Francisco. + +May. FBI interviews John Perry Barlow re NuPrometheus case. + +June. Mitch Kapor and Barlow found Electronic Frontier Foundation; + Barlow publishes CRIME AND PUZZLEMENT manifesto. + +July 24-27. Trial of Knight Lightning. + +1991 + +February. CPSR Roundtable in Washington, D.C. + +March 25-28. Computers, Freedom and Privacy conference in San Francisco. + +May 1. Electronic Frontier Foundation, Steve Jackson, + and others file suit against members of Chicago Task Force. + +July 1-2. Switching station phone software crash affects + Washington, Los Angeles, Pittsburgh, San Francisco. + +September 17. AT&T phone crash affects New York City and three airports. + + + + +Introduction + +This is a book about cops, and wild teenage whiz-kids, and lawyers, +and hairy-eyed anarchists, and industrial technicians, and hippies, +and high-tech millionaires, and game hobbyists, and computer security +experts, and Secret Service agents, and grifters, and thieves. + +This book is about the electronic frontier of the 1990s. +It concerns activities that take place inside computers +and over telephone lines. + +A science fiction writer coined the useful term "cyberspace" in 1982, +but the territory in question, the electronic frontier, is about +a hundred and thirty years old. Cyberspace is the "place" where +a telephone conversation appears to occur. Not inside your actual phone, +the plastic device on your desk. Not inside the other person's phone, +in some other city. THE PLACE BETWEEN the phones. The indefinite +place OUT THERE, where the two of you, two human beings, +actually meet and communicate. + +Although it is not exactly "real," "cyberspace" is a genuine place. +Things happen there that have very genuine consequences. This "place" +is not "real," but it is serious, it is earnest. Tens of thousands +of people have dedicated their lives to it, to the public service +of public communication by wire and electronics. + +People have worked on this "frontier" for generations now. +Some people became rich and famous from their efforts there. +Some just played in it, as hobbyists. Others soberly pondered it, +and wrote about it, and regulated it, and negotiated over it in +international forums, and sued one another about it, in gigantic, +epic court battles that lasted for years. And almost since +the beginning, some people have committed crimes in this place. + +But in the past twenty years, this electrical "space," +which was once thin and dark and one-dimensional--little more +than a narrow speaking-tube, stretching from phone to phone-- +has flung itself open like a gigantic jack-in-the-box. +Light has flooded upon it, the eerie light of the glowing computer screen. +This dark electric netherworld has become a vast flowering electronic landscape. +Since the 1960s, the world of the telephone has cross-bred itself +with computers and television, and though there is still no substance +to cyberspace, nothing you can handle, it has a strange kind +of physicality now. It makes good sense today to talk of cyberspace +as a place all its own. + +Because people live in it now. Not just a few people, +not just a few technicians and eccentrics, but thousands +of people, quite normal people. And not just for a little while, +either, but for hours straight, over weeks, and months, +and years. Cyberspace today is a "Net," a "Matrix," +international in scope and growing swiftly and steadily. +It's growing in size, and wealth, and political importance. + +People are making entire careers in modern cyberspace. +Scientists and technicians, of course; they've been there +for twenty years now. But increasingly, cyberspace +is filling with journalists and doctors and lawyers +and artists and clerks. Civil servants make their +careers there now, "on-line" in vast government data-banks; +and so do spies, industrial, political, and just plain snoops; +and so do police, at least a few of them. And there are children +living there now. + +People have met there and been married there. +There are entire living communities in cyberspace today; +chattering, gossiping, planning, conferring and scheming, +leaving one another voice-mail and electronic mail, +giving one another big weightless chunks of valuable data, +both legitimate and illegitimate. They busily pass one another +computer software and the occasional festering computer virus. + +We do not really understand how to live in cyberspace yet. +We are feeling our way into it, blundering about. +That is not surprising. Our lives in the physical world, +the "real" world, are also far from perfect, despite a lot more practice. +Human lives, real lives, are imperfect by their nature, and there are +human beings in cyberspace. The way we live in cyberspace is +a funhouse mirror of the way we live in the real world. +We take both our advantages and our troubles with us. + +This book is about trouble in cyberspace. +Specifically, this book is about certain strange events in +the year 1990, an unprecedented and startling year for the +the growing world of computerized communications. + +In 1990 there came a nationwide crackdown on illicit +computer hackers, with arrests, criminal charges, +one dramatic show-trial, several guilty pleas, and +huge confiscations of data and equipment all over the USA. + +The Hacker Crackdown of 1990 was larger, better organized, +more deliberate, and more resolute than any previous effort +in the brave new world of computer crime. The U.S. Secret Service, +private telephone security, and state and local law enforcement groups +across the country all joined forces in a determined attempt to break +the back of America's electronic underground. It was a fascinating +effort, with very mixed results. + +The Hacker Crackdown had another unprecedented effect; +it spurred the creation, within "the computer community," +of the Electronic Frontier Foundation, a new and very odd +interest group, fiercely dedicated to the establishment +and preservation of electronic civil liberties. The crackdown, +remarkable in itself, has created a melee of debate over electronic crime, +punishment, freedom of the press, and issues of search and seizure. +Politics has entered cyberspace. Where people go, politics follow. + +This is the story of the people of cyberspace. + + + +PART ONE: Crashing the System + +On January 15, 1990, AT&T's long-distance telephone switching system crashed. + +This was a strange, dire, huge event. Sixty thousand people lost +their telephone service completely. During the nine long hours +of frantic effort that it took to restore service, some seventy million +telephone calls went uncompleted. + +Losses of service, known as "outages" in the telco trade, +are a known and accepted hazard of the telephone business. +Hurricanes hit, and phone cables get snapped by the thousands. +Earthquakes wrench through buried fiber-optic lines. +Switching stations catch fire and burn to the ground. +These things do happen. There are contingency plans for them, +and decades of experience in dealing with them. +But the Crash of January 15 was unprecedented. +It was unbelievably huge, and it occurred for +no apparent physical reason. + +The crash started on a Monday afternoon in a single +switching-station in Manhattan. But, unlike any merely +physical damage, it spread and spread. Station after +station across America collapsed in a chain reaction, +until fully half of AT&T's network had gone haywire +and the remaining half was hard-put to handle the overflow. + +Within nine hours, AT&T software engineers more or less +understood what had caused the crash. Replicating the +problem exactly, poring over software line by line, +took them a couple of weeks. But because it was hard +to understand technically, the full truth of the matter +and its implications were not widely and thoroughly aired +and explained. The root cause of the crash remained obscure, +surrounded by rumor and fear. + +The crash was a grave corporate embarrassment. +The "culprit" was a bug in AT&T's own software--not the +sort of admission the telecommunications giant wanted +to make, especially in the face of increasing competition. +Still, the truth WAS told, in the baffling technical terms +necessary to explain it. + +Somehow the explanation failed to persuade +American law enforcement officials and even telephone +corporate security personnel. These people were not +technical experts or software wizards, and they had their +own suspicions about the cause of this disaster. + +The police and telco security had important sources +of information denied to mere software engineers. +They had informants in the computer underground and +years of experience in dealing with high-tech rascality +that seemed to grow ever more sophisticated. +For years they had been expecting a direct and +savage attack against the American national telephone system. +And with the Crash of January 15--the first month of a +new, high-tech decade--their predictions, fears, +and suspicions seemed at last to have entered the real world. +A world where the telephone system had not merely crashed, +but, quite likely, BEEN crashed--by "hackers." + +The crash created a large dark cloud of suspicion +that would color certain people's assumptions and actions +for months. The fact that it took place in the realm of +software was suspicious on its face. The fact that it +occurred on Martin Luther King Day, still the most +politically touchy of American holidays, made it more +suspicious yet. + +The Crash of January 15 gave the Hacker Crackdown +its sense of edge and its sweaty urgency. It made people, +powerful people in positions of public authority, +willing to believe the worst. And, most fatally, +it helped to give investigators a willingness +to take extreme measures and the determination +to preserve almost total secrecy. + +An obscure software fault in an aging switching system +in New York was to lead to a chain reaction of legal +and constitutional trouble all across the country. + +# + +Like the crash in the telephone system, this chain reaction +was ready and waiting to happen. During the 1980s, +the American legal system was extensively patched +to deal with the novel issues of computer crime. +There was, for instance, the Electronic Communications +Privacy Act of 1986 (eloquently described as "a stinking mess" +by a prominent law enforcement official). And there was the +draconian Computer Fraud and Abuse Act of 1986, passed unanimously +by the United States Senate, which later would reveal +a large number of flaws. Extensive, well-meant efforts +had been made to keep the legal system up to date. +But in the day-to-day grind of the real world, +even the most elegant software tends to crumble +and suddenly reveal its hidden bugs. + +Like the advancing telephone system, the American legal system +was certainly not ruined by its temporary crash; but for those +caught under the weight of the collapsing system, life became +a series of blackouts and anomalies. + +In order to understand why these weird events occurred, +both in the world of technology and in the world of law, +it's not enough to understand the merely technical problems. +We will get to those; but first and foremost, we must try +to understand the telephone, and the business of telephones, +and the community of human beings that telephones have created. + +# + +Technologies have life cycles, like cities do, +like institutions do, like laws and governments do. + +The first stage of any technology is the Question +Mark, often known as the "Golden Vaporware" stage. +At this early point, the technology is only a phantom, +a mere gleam in the inventor's eye. One such inventor +was a speech teacher and electrical tinkerer named +Alexander Graham Bell. + +Bell's early inventions, while ingenious, failed to move the world. +In 1863, the teenage Bell and his brother Melville made an artificial +talking mechanism out of wood, rubber, gutta-percha, and tin. +This weird device had a rubber-covered "tongue" made of movable +wooden segments, with vibrating rubber "vocal cords," and +rubber "lips" and "cheeks." While Melville puffed a bellows +into a tin tube, imitating the lungs, young Alec Bell would +manipulate the "lips," "teeth," and "tongue," causing the thing +to emit high-pitched falsetto gibberish. + +Another would-be technical breakthrough was the Bell "phonautograph" +of 1874, actually made out of a human cadaver's ear. Clamped into place +on a tripod, this grisly gadget drew sound-wave images on smoked glass +through a thin straw glued to its vibrating earbones. + +By 1875, Bell had learned to produce audible sounds--ugly shrieks +and squawks--by using magnets, diaphragms, and electrical current. + +Most "Golden Vaporware" technologies go nowhere. + +But the second stage of technology is the Rising Star, +or, the "Goofy Prototype," stage. The telephone, Bell's +most ambitious gadget yet, reached this stage on March +10, 1876. On that great day, Alexander Graham Bell +became the first person to transmit intelligible human +speech electrically. As it happened, young Professor Bell, +industriously tinkering in his Boston lab, had spattered +his trousers with acid. His assistant, Mr. Watson, +heard his cry for help--over Bell's experimental +audio-telegraph. This was an event without precedent. + +Technologies in their "Goofy Prototype" stage rarely +work very well. They're experimental, and therefore +half- baked and rather frazzled. The prototype may +be attractive and novel, and it does look as if it ought +to be good for something-or-other. But nobody, including +the inventor, is quite sure what. Inventors, and speculators, +and pundits may have very firm ideas about its potential +use, but those ideas are often very wrong. + +The natural habitat of the Goofy Prototype is in trade shows +and in the popular press. Infant technologies need publicity +and investment money like a tottering calf need milk. +This was very true of Bell's machine. To raise research and +development money, Bell toured with his device as a stage attraction. + +Contemporary press reports of the stage debut of the telephone +showed pleased astonishment mixed with considerable dread. +Bell's stage telephone was a large wooden box with a crude +speaker-nozzle, the whole contraption about the size and shape +of an overgrown Brownie camera. Its buzzing steel soundplate, +pumped up by powerful electromagnets, was loud enough to fill +an auditorium. Bell's assistant Mr. Watson, who could manage +on the keyboards fairly well, kicked in by playing the organ +from distant rooms, and, later, distant cities. This feat was +considered marvellous, but very eerie indeed. + +Bell's original notion for the telephone, an idea promoted +for a couple of years, was that it would become a mass medium. +We might recognize Bell's idea today as something close to modern +"cable radio." Telephones at a central source would transmit music, +Sunday sermons, and important public speeches to a paying network +of wired-up subscribers. + +At the time, most people thought this notion made good sense. +In fact, Bell's idea was workable. In Hungary, this philosophy +of the telephone was successfully put into everyday practice. +In Budapest, for decades, from 1893 until after World War I, +there was a government-run information service called +"Telefon Hirmondo-." Hirmondo- was a centralized source +of news and entertainment and culture, including stock reports, +plays, concerts, and novels read aloud. At certain hours +of the day, the phone would ring, you would plug in +a loudspeaker for the use of the family, and Telefon +Hirmondo- would be on the air--or rather, on the phone. + +Hirmondo- is dead tech today, but Hirmondo- might be considered +a spiritual ancestor of the modern telephone-accessed computer +data services, such as CompuServe, GEnie or Prodigy. +The principle behind Hirmondo- is also not too far from computer +"bulletin- board systems" or BBS's, which arrived in the late 1970s, +spread rapidly across America, and will figure largely in this book. + +We are used to using telephones for individual person-to-person speech, +because we are used to the Bell system. But this was just one possibility +among many. Communication networks are very flexible and protean, +especially when their hardware becomes sufficiently advanced. +They can be put to all kinds of uses. And they have been-- +and they will be. + +Bell's telephone was bound for glory, but this was a combination +of political decisions, canny infighting in court, inspired industrial +leadership, receptive local conditions and outright good luck. +Much the same is true of communications systems today. + +As Bell and his backers struggled to install their newfangled system +in the real world of nineteenth-century New England, they had to fight +against skepticism and industrial rivalry. There was already a strong +electrical communications network present in America: the telegraph. +The head of the Western Union telegraph system dismissed Bell's prototype +as "an electrical toy" and refused to buy the rights to Bell's patent. +The telephone, it seemed, might be all right as a parlor entertainment-- +but not for serious business. + +Telegrams, unlike mere telephones, left a permanent physical record +of their messages. Telegrams, unlike telephones, could be answered +whenever the recipient had time and convenience. And the telegram +had a much longer distance-range than Bell's early telephone. +These factors made telegraphy seem a much more sound and businesslike +technology--at least to some. + +The telegraph system was huge, and well-entrenched. +In 1876, the United States had 214,000 miles of telegraph wire, +and 8500 telegraph offices. There were specialized telegraphs +for businesses and stock traders, government, police and fire departments. +And Bell's "toy" was best known as a stage-magic musical device. + +The third stage of technology is known as the "Cash Cow" stage. +In the "cash cow" stage, a technology finds its place in the world, +and matures, and becomes settled and productive. After a year or so, +Alexander Graham Bell and his capitalist backers concluded that +eerie music piped from nineteenth-century cyberspace was not the real +selling-point of his invention. Instead, the telephone was about speech-- +individual, personal speech, the human voice, human conversation and +human interaction. The telephone was not to be managed from any centralized +broadcast center. It was to be a personal, intimate technology. + +When you picked up a telephone, you were not absorbing +the cold output of a machine--you were speaking to another human being. +Once people realized this, their instinctive dread of the telephone +as an eerie, unnatural device, swiftly vanished. A "telephone call" +was not a "call" from a "telephone" itself, but a call from another +human being, someone you would generally know and recognize. +The real point was not what the machine could do for you (or to you), +but what you yourself, a person and citizen, could do THROUGH the machine. +This decision on the part of the young Bell Company was absolutely vital. + +The first telephone networks went up around Boston--mostly among +the technically curious and the well-to-do (much the same segment +of the American populace that, a hundred years later, would be +buying personal computers). Entrenched backers of the telegraph +continued to scoff. + +But in January 1878, a disaster made the telephone famous. +A train crashed in Tarriffville, Connecticut. Forward-looking +doctors in the nearby city of Hartford had had Bell's +"speaking telephone" installed. An alert local druggist +was able to telephone an entire community of local doctors, +who rushed to the site to give aid. The disaster, as disasters do, +aroused intense press coverage. The phone had proven its usefulness +in the real world. + +After Tarriffville, the telephone network spread like crabgrass. +By 1890 it was all over New England. By '93, out to Chicago. +By '97, into Minnesota, Nebraska and Texas. By 1904 it was +all over the continent. + +The telephone had become a mature technology. Professor Bell +(now generally known as "Dr. Bell" despite his lack of a formal degree) +became quite wealthy. He lost interest in the tedious day-to-day business +muddle of the booming telephone network, and gratefully returned +his attention to creatively hacking-around in his various laboratories, +which were now much larger, better-ventilated, and gratifyingly +better-equipped. Bell was never to have another great inventive success, +though his speculations and prototypes anticipated fiber-optic transmission, +manned flight, sonar, hydrofoil ships, tetrahedral construction, and +Montessori education. The "decibel," the standard scientific measure +of sound intensity, was named after Bell. + +Not all Bell's vaporware notions were inspired. He was fascinated +by human eugenics. He also spent many years developing a weird personal +system of astrophysics in which gravity did not exist. + +Bell was a definite eccentric. He was something of a hypochondriac, +and throughout his life he habitually stayed up until four A.M., +refusing to rise before noon. But Bell had accomplished a great feat; +he was an idol of millions and his influence, wealth, and great +personal charm, combined with his eccentricity, made him something +of a loose cannon on deck. Bell maintained a thriving scientific +salon in his winter mansion in Washington, D.C., which gave him +considerable backstage influence in governmental and scientific circles. +He was a major financial backer of the the magazines Science and +National Geographic, both still flourishing today as important organs +of the American scientific establishment. + +Bell's companion Thomas Watson, similarly wealthy and similarly odd, +became the ardent political disciple of a 19th-century science-fiction writer +and would-be social reformer, Edward Bellamy. Watson also trod the boards +briefly as a Shakespearian actor. + +There would never be another Alexander Graham Bell, +but in years to come there would be surprising numbers +of people like him. Bell was a prototype of the +high-tech entrepreneur. High-tech entrepreneurs will +play a very prominent role in this book: not merely as +technicians and businessmen, but as pioneers of the +technical frontier, who can carry the power and prestige +they derive from high-technology into the political and +social arena. + +Like later entrepreneurs, Bell was fierce in defense of +his own technological territory. As the telephone began to +flourish, Bell was soon involved in violent lawsuits in the +defense of his patents. Bell's Boston lawyers were +excellent, however, and Bell himself, as an elocution +teacher and gifted public speaker, was a devastatingly +effective legal witness. In the eighteen years of Bell's patents, +the Bell company was involved in six hundred separate lawsuits. +The legal records printed filled 149 volumes. The Bell Company +won every single suit. + +After Bell's exclusive patents expired, rival telephone +companies sprang up all over America. Bell's company, +American Bell Telephone, was soon in deep trouble. +In 1907, American Bell Telephone fell into the hands of the +rather sinister J.P. Morgan financial cartel, robber-baron +speculators who dominated Wall Street. + +At this point, history might have taken a different turn. +American might well have been served forever by a patchwork +of locally owned telephone companies. Many state politicians +and local businessmen considered this an excellent solution. + +But the new Bell holding company, American Telephone and Telegraph +or AT&T, put in a new man at the helm, a visionary industrialist +named Theodore Vail. Vail, a former Post Office manager, +understood large organizations and had an innate feeling +for the nature of large-scale communications. Vail quickly +saw to it that AT&T seized the technological edge once again. +The Pupin and Campbell "loading coil," and the deForest +"audion," are both extinct technology today, but in 1913 +they gave Vail's company the best LONG-DISTANCE lines +ever built. By controlling long-distance--the links +between, and over, and above the smaller local phone +companies--AT&T swiftly gained the whip-hand over them, +and was soon devouring them right and left. + +Vail plowed the profits back into research and development, +starting the Bell tradition of huge-scale and brilliant +industrial research. + +Technically and financially, AT&T gradually steamrollered +the opposition. Independent telephone companies never +became entirely extinct, and hundreds of them flourish today. +But Vail's AT&T became the supreme communications company. +At one point, Vail's AT&T bought Western Union itself, +the very company that had derided Bell's telephone as a "toy." +Vail thoroughly reformed Western Union's hidebound business +along his modern principles; but when the federal government +grew anxious at this centralization of power, Vail politely +gave Western Union back. + +This centralizing process was not unique. Very similar +events had happened in American steel, oil, and railroads. +But AT&T, unlike the other companies, was to remain supreme. +The monopoly robber-barons of those other industries +were humbled and shattered by government trust-busting. + +Vail, the former Post Office official, was quite willing +to accommodate the US government; in fact he would +forge an active alliance with it. AT&T would become +almost a wing of the American government, almost +another Post Office--though not quite. AT&T would +willingly submit to federal regulation, but in return, +it would use the government's regulators as its own police, +who would keep out competitors and assure the Bell +system's profits and preeminence. + +This was the second birth--the political birth--of the +American telephone system. Vail's arrangement was to +persist, with vast success, for many decades, until 1982. +His system was an odd kind of American industrial socialism. +It was born at about the same time as Leninist Communism, +and it lasted almost as long--and, it must be admitted, +to considerably better effect. + +Vail's system worked. Except perhaps for aerospace, +there has been no technology more thoroughly dominated +by Americans than the telephone. The telephone was +seen from the beginning as a quintessentially American +technology. Bell's policy, and the policy of Theodore Vail, +was a profoundly democratic policy of UNIVERSAL ACCESS. +Vail's famous corporate slogan, "One Policy, One System, +Universal Service," was a political slogan, with a very +American ring to it. + +The American telephone was not to become the specialized tool +of government or business, but a general public utility. +At first, it was true, only the wealthy could afford +private telephones, and Bell's company pursued the +business markets primarily. The American phone system +was a capitalist effort, meant to make money; it was not a charity. +But from the first, almost all communities with telephone service +had public telephones. And many stores--especially drugstores-- +offered public use of their phones. You might not own a telephone-- +but you could always get into the system, if you really needed to. + +There was nothing inevitable about this decision to make telephones +"public" and "universal." Vail's system involved a profound act +of trust in the public. This decision was a political one, +informed by the basic values of the American republic. +The situation might have been very different; +and in other countries, under other systems, +it certainly was. + +Joseph Stalin, for instance, vetoed plans for a Soviet +phone system soon after the Bolshevik revolution. +Stalin was certain that publicly accessible telephones +would become instruments of anti-Soviet counterrevolution +and conspiracy. (He was probably right.) When telephones +did arrive in the Soviet Union, they would be instruments +of Party authority, and always heavily tapped. (Alexander +Solzhenitsyn's prison-camp novel The First Circle +describes efforts to develop a phone system more suited +to Stalinist purposes.) + +France, with its tradition of rational centralized government, +had fought bitterly even against the electric telegraph, +which seemed to the French entirely too anarchical and frivolous. +For decades, nineteenth-century France communicated via the +"visual telegraph," a nation-spanning, government-owned semaphore +system of huge stone towers that signalled from hilltops, +across vast distances, with big windmill-like arms. +In 1846, one Dr. Barbay, a semaphore enthusiast, +memorably uttered an early version of what might be called +"the security expert's argument" against the open media. + +"No, the electric telegraph is not a sound invention. +It will always be at the mercy of the slightest disruption, +wild youths, drunkards, bums, etc. . . . The electric telegraph +meets those destructive elements with only a few meters of wire +over which supervision is impossible. A single man could, +without being seen, cut the telegraph wires leading to Paris, +and in twenty-four hours cut in ten different places the wires +of the same line, without being arrested. The visual telegraph, +on the contrary, has its towers, its high walls, its gates +well-guarded from inside by strong armed men. Yes, I declare, +substitution of the electric telegraph for the visual one +is a dreadful measure, a truly idiotic act." + +Dr. Barbay and his high-security stone machines +were eventually unsuccessful, but his argument-- +that communication exists for the safety and convenience +of the state, and must be carefully protected from the wild +boys and the gutter rabble who might want to crash the +system--would be heard again and again. + +When the French telephone system finally did arrive, +its snarled inadequacy was to be notorious. Devotees +of the American Bell System often recommended a trip +to France, for skeptics. + +In Edwardian Britain, issues of class and privacy +were a ball-and-chain for telephonic progress. It was +considered outrageous that anyone--any wild fool off +the street--could simply barge bellowing into one's office +or home, preceded only by the ringing of a telephone bell. +In Britain, phones were tolerated for the use of business, +but private phones tended be stuffed away into closets, +smoking rooms, or servants' quarters. Telephone operators +were resented in Britain because they did not seem to +"know their place." And no one of breeding would print +a telephone number on a business card; this seemed a crass +attempt to make the acquaintance of strangers. + +But phone access in America was to become a popular right; +something like universal suffrage, only more so. +American women could not yet vote when the phone system +came through; yet from the beginning American women +doted on the telephone. This "feminization" of the +American telephone was often commented on by foreigners. +Phones in America were not censored or stiff or formalized; +they were social, private, intimate, and domestic. +In America, Mother's Day is by far the busiest day +of the year for the phone network. + +The early telephone companies, and especially AT&T, +were among the foremost employers of American women. +They employed the daughters of the American middle-class +in great armies: in 1891, eight thousand women; by 1946, +almost a quarter of a million. Women seemed to enjoy +telephone work; it was respectable, it was steady, +it paid fairly well as women's work went, and--not least-- +it seemed a genuine contribution to the social good +of the community. Women found Vail's ideal of public +service attractive. This was especially true in rural areas, +where women operators, running extensive rural party-lines, +enjoyed considerable social power. The operator knew everyone +on the party-line, and everyone knew her. + +Although Bell himself was an ardent suffragist, the +telephone company did not employ women for the sake of +advancing female liberation. AT&T did this for sound +commercial reasons. The first telephone operators of +the Bell system were not women, but teenage American boys. +They were telegraphic messenger boys (a group about to +be rendered technically obsolescent), who swept up +around the phone office, dunned customers for bills, +and made phone connections on the switchboard, +all on the cheap. + +Within the very first year of operation, 1878, +Bell's company learned a sharp lesson about combining +teenage boys and telephone switchboards. Putting +teenage boys in charge of the phone system brought swift +and consistent disaster. Bell's chief engineer described them +as "Wild Indians." The boys were openly rude to customers. +They talked back to subscribers, saucing off, +uttering facetious remarks, and generally giving lip. +The rascals took Saint Patrick's Day off without permission. +And worst of all they played clever tricks with +the switchboard plugs: disconnecting calls, crossing lines +so that customers found themselves talking to strangers, +and so forth. + +This combination of power, technical mastery, and effective +anonymity seemed to act like catnip on teenage boys. + +This wild-kid-on-the-wires phenomenon was not confined to +the USA; from the beginning, the same was true of the British +phone system. An early British commentator kindly remarked: +"No doubt boys in their teens found the work not a little irksome, +and it is also highly probable that under the early conditions +of employment the adventurous and inquisitive spirits of which +the average healthy boy of that age is possessed, were not always +conducive to the best attention being given to the wants +of the telephone subscribers." + +So the boys were flung off the system--or at least, +deprived of control of the switchboard. But the +"adventurous and inquisitive spirits" of the teenage boys +would be heard from in the world of telephony, again and again. + +The fourth stage in the technological life-cycle is death: +"the Dog," dead tech. The telephone has so far avoided this fate. +On the contrary, it is thriving, still spreading, still evolving, +and at increasing speed. + +The telephone has achieved a rare and exalted state for a +technological artifact: it has become a HOUSEHOLD OBJECT. +The telephone, like the clock, like pen and paper, +like kitchen utensils and running water, has become +a technology that is visible only by its absence. +The telephone is technologically transparent. +The global telephone system is the largest and most +complex machine in the world, yet it is easy to use. +More remarkable yet, the telephone is almost entirely +physically safe for the user. + +For the average citizen in the 1870s, the telephone +was weirder, more shocking, more "high-tech" and +harder to comprehend, than the most outrageous stunts +of advanced computing for us Americans in the 1990s. +In trying to understand what is happening to us today, +with our bulletin-board systems, direct overseas dialling, +fiber-optic transmissions, computer viruses, hacking stunts, +and a vivid tangle of new laws and new crimes, it is important +to realize that our society has been through a similar challenge before-- +and that, all in all, we did rather well by it. + +Bell's stage telephone seemed bizarre at first. But the +sensations of weirdness vanished quickly, once people began +to hear the familiar voices of relatives and friends, +in their own homes on their own telephones. The telephone +changed from a fearsome high-tech totem to an everyday pillar +of human community. + +This has also happened, and is still happening, +to computer networks. Computer networks such as +NSFnet, BITnet, USENET, JANET, are technically +advanced, intimidating, and much harder to use than +telephones. Even the popular, commercial computer +networks, such as GEnie, Prodigy, and CompuServe, +cause much head-scratching and have been described +as "user-hateful." Nevertheless they too are changing +from fancy high-tech items into everyday sources +of human community. + +The words "community" and "communication" have +the same root. Wherever you put a communications +network, you put a community as well. And whenever +you TAKE AWAY that network--confiscate it, outlaw it, +crash it, raise its price beyond affordability-- +then you hurt that community. + +Communities will fight to defend themselves. People will fight harder +and more bitterly to defend their communities, than they will fight +to defend their own individual selves. And this is very true +of the "electronic community" that arose around computer networks +in the 1980s--or rather, the VARIOUS electronic communities, +in telephony, law enforcement, computing, and the digital +underground that, by the year 1990, were raiding, rallying, +arresting, suing, jailing, fining and issuing angry manifestos. + +None of the events of 1990 were entirely new. +Nothing happened in 1990 that did not have some kind +of earlier and more understandable precedent. What gave +the Hacker Crackdown its new sense of gravity and +importance was the feeling--the COMMUNITY feeling-- +that the political stakes had been raised; that trouble +in cyberspace was no longer mere mischief or inconclusive +skirmishing, but a genuine fight over genuine issues, +a fight for community survival and the shape of the future. + +These electronic communities, having flourished throughout +the 1980s, were becoming aware of themselves, and increasingly, +becoming aware of other, rival communities. Worries were +sprouting up right and left, with complaints, rumors, +uneasy speculations. But it would take a catalyst, a shock, +to make the new world evident. Like Bell's great publicity break, +the Tarriffville Rail Disaster of January 1878, +it would take a cause celebre. + +That cause was the AT&T Crash of January 15, 1990. +After the Crash, the wounded and anxious telephone +community would come out fighting hard. + +# + +The community of telephone technicians, engineers, operators +and researchers is the oldest community in cyberspace. +These are the veterans, the most developed group, +the richest, the most respectable, in most ways the most powerful. +Whole generations have come and gone since Alexander Graham Bell's day, +but the community he founded survives; people work for the phone system +today whose great-grandparents worked for the phone system. +Its specialty magazines, such as Telephony, AT&T Technical Journal, +Telephone Engineer and Management, are decades old; +they make computer publications like Macworld and PC Week +look like amateur johnny-come-latelies. + +And the phone companies take no back seat in high-technology, either. +Other companies' industrial researchers may have won new markets; +but the researchers of Bell Labs have won SEVEN NOBEL PRIZES. +One potent device that Bell Labs originated, the transistor, +has created entire GROUPS of industries. Bell Labs are +world-famous for generating "a patent a day," and have even +made vital discoveries in astronomy, physics and cosmology. + +Throughout its seventy-year history, "Ma Bell" was not so much +a company as a way of life. Until the cataclysmic divestiture +of the 1980s, Ma Bell was perhaps the ultimate maternalist mega-employer. +The AT&T corporate image was the "gentle giant," "the voice with a smile," +a vaguely socialist-realist world of cleanshaven linemen in shiny helmets +and blandly pretty phone-girls in headsets and nylons. Bell System +employees were famous as rock-ribbed Kiwanis and Rotary members, +Little-League enthusiasts, school-board people. + +During the long heyday of Ma Bell, the Bell employee corps +were nurtured top-to-bottom on a corporate ethos of public service. +There was good money in Bell, but Bell was not ABOUT money; +Bell used public relations, but never mere marketeering. +People went into the Bell System for a good life, +and they had a good life. But it was not mere money +that led Bell people out in the midst of storms and earthquakes +to fight with toppled phone-poles, to wade in flooded manholes, +to pull the red-eyed graveyard-shift over collapsing switching-systems. +The Bell ethic was the electrical equivalent of the postman's: +neither rain, nor snow, nor gloom of night would stop these couriers. + +It is easy to be cynical about this, as it is easy to be +cynical about any political or social system; but cynicism +does not change the fact that thousands of people took +these ideals very seriously. And some still do. + +The Bell ethos was about public service; and that was +gratifying; but it was also about private POWER, and that +was gratifying too. As a corporation, Bell was very special. +Bell was privileged. Bell had snuggled up close to the state. +In fact, Bell was as close to government as you could get in +America and still make a whole lot of legitimate money. + +But unlike other companies, Bell was above and beyond +the vulgar commercial fray. Through its regional operating companies, +Bell was omnipresent, local, and intimate, all over America; +but the central ivory towers at its corporate heart were the +tallest and the ivoriest around. + +There were other phone companies in America, to be sure; +the so-called independents. Rural cooperatives, mostly; +small fry, mostly tolerated, sometimes warred upon. +For many decades, "independent" American phone companies +lived in fear and loathing of the official Bell monopoly +(or the "Bell Octopus," as Ma Bell's nineteenth-century +enemies described her in many angry newspaper manifestos). +Some few of these independent entrepreneurs, while legally +in the wrong, fought so bitterly against the Octopus +that their illegal phone networks were cast into the street +by Bell agents and publicly burned. + +The pure technical sweetness of the Bell System gave its operators, +inventors and engineers a deeply satisfying sense of power and mastery. +They had devoted their lives to improving this vast nation-spanning machine; +over years, whole human lives, they had watched it improve and grow. +It was like a great technological temple. They were an elite, +and they knew it--even if others did not; in fact, they felt +even more powerful BECAUSE others did not understand. + +The deep attraction of this sensation of elite technical power +should never be underestimated. "Technical power" is not for everybody; +for many people it simply has no charm at all. But for some people, +it becomes the core of their lives. For a few, it is overwhelming, +obsessive; it becomes something close to an addiction. People--especially +clever teenage boys whose lives are otherwise mostly powerless and put-upon +--love this sensation of secret power, and are willing to do all sorts +of amazing things to achieve it. The technical POWER of electronics +has motivated many strange acts detailed in this book, which would +otherwise be inexplicable. + +So Bell had power beyond mere capitalism. The Bell service ethos worked, +and was often propagandized, in a rather saccharine fashion. Over the decades, +people slowly grew tired of this. And then, openly impatient with it. +By the early 1980s, Ma Bell was to find herself with scarcely a real friend +in the world. Vail's industrial socialism had become hopelessly +out-of-fashion politically. Bell would be punished for that. +And that punishment would fall harshly upon the people of the +telephone community. + +# + +In 1983, Ma Bell was dismantled by federal court action. +The pieces of Bell are now separate corporate entities. +The core of the company became AT&T Communications, +and also AT&T Industries (formerly Western Electric, +Bell's manufacturing arm). AT&T Bell Labs became Bell +Communications Research, Bellcore. Then there are the +Regional Bell Operating Companies, or RBOCs, pronounced "arbocks." + +Bell was a titan and even these regional chunks are gigantic enterprises: +Fortune 50 companies with plenty of wealth and power behind them. +But the clean lines of "One Policy, One System, Universal Service" +have been shattered, apparently forever. + +The "One Policy" of the early Reagan Administration was to +shatter a system that smacked of noncompetitive socialism. +Since that time, there has been no real telephone "policy" +on the federal level. Despite the breakup, the remnants +of Bell have never been set free to compete in the open marketplace. + +The RBOCs are still very heavily regulated, but not from the top. +Instead, they struggle politically, economically and legally, +in what seems an endless turmoil, in a patchwork of overlapping federal +and state jurisdictions. Increasingly, like other major American corporations, +the RBOCs are becoming multinational, acquiring important commercial interests +in Europe, Latin America, and the Pacific Rim. But this, too, adds to their +legal and political predicament. + +The people of what used to be Ma Bell are not happy about their fate. +They feel ill-used. They might have been grudgingly willing to make +a full transition to the free market; to become just companies amid +other companies. But this never happened. Instead, AT&T and the RBOCS +("the Baby Bells") feel themselves wrenched from side to side by state +regulators, by Congress, by the FCC, and especially by the federal court +of Judge Harold Greene, the magistrate who ordered the Bell breakup +and who has been the de facto czar of American telecommunications +ever since 1983. + +Bell people feel that they exist in a kind of paralegal limbo today. +They don't understand what's demanded of them. If it's "service," +why aren't they treated like a public service? And if it's money, +then why aren't they free to compete for it? No one seems to know, +really. Those who claim to know keep changing their minds. +Nobody in authority seems willing to grasp the nettle for once and all. + +Telephone people from other countries are amazed by the +American telephone system today. Not that it works so well; +for nowadays even the French telephone system works, more or less. +They are amazed that the American telephone system STILL works +AT ALL, under these strange conditions. + +Bell's "One System" of long-distance service is now only about +eighty percent of a system, with the remainder held by Sprint, MCI, +and the midget long-distance companies. Ugly wars over dubious +corporate practices such as "slamming" (an underhanded method +of snitching clients from rivals) break out with some regularity +in the realm of long-distance service. The battle to break Bell's +long-distance monopoly was long and ugly, and since the breakup +the battlefield has not become much prettier. AT&T's famous +shame-and-blame advertisements, which emphasized the shoddy work +and purported ethical shadiness of their competitors, were much +remarked on for their studied psychological cruelty. + +There is much bad blood in this industry, and much +long-treasured resentment. AT&T's post-breakup +corporate logo, a striped sphere, is known in the +industry as the "Death Star" (a reference from the movie +Star Wars, in which the "Death Star" was the spherical +high- tech fortress of the harsh-breathing imperial ultra-baddie, +Darth Vader.) Even AT&T employees are less than thrilled +by the Death Star. A popular (though banned) T-shirt among +AT&T employees bears the old-fashioned Bell logo of the Bell System, +plus the newfangled striped sphere, with the before-and-after comments: +"This is your brain--This is your brain on drugs!" AT&T made a very +well-financed and determined effort to break into the personal +computer market; it was disastrous, and telco computer experts +are derisively known by their competitors as "the pole-climbers." +AT&T and the Baby Bell arbocks still seem to have few friends. + +Under conditions of sharp commercial competition, a crash like +that of January 15, 1990 was a major embarrassment to AT&T. +It was a direct blow against their much-treasured reputation +for reliability. Within days of the crash AT&T's +Chief Executive Officer, Bob Allen, officially apologized, +in terms of deeply pained humility: + +"AT&T had a major service disruption last Monday. +We didn't live up to our own standards of quality, +and we didn't live up to yours. It's as simple as that. +And that's not acceptable to us. Or to you. . . . +We understand how much people have come to depend +upon AT&T service, so our AT&T Bell Laboratories scientists +and our network engineers are doing everything possible +to guard against a recurrence. . . . We know there's no way +to make up for the inconvenience this problem may have caused you." + +Mr Allen's "open letter to customers" was printed in lavish ads +all over the country: in the Wall Street Journal, USA Today, +New York Times, Los Angeles Times, Chicago Tribune, +Philadelphia Inquirer, San Francisco Chronicle Examiner, +Boston Globe, Dallas Morning News, Detroit Free Press, +Washington Post, Houston Chronicle, Cleveland Plain Dealer, +Atlanta Journal Constitution, Minneapolis Star Tribune, +St. Paul Pioneer Press Dispatch, Seattle Times/Post Intelligencer, +Tacoma News Tribune, Miami Herald, Pittsburgh Press, +St. Louis Post Dispatch, Denver Post, Phoenix Republic Gazette +and Tampa Tribune. + +In another press release, AT&T went to some pains to suggest +that this "software glitch" might have happened just as easily to MCI, +although, in fact, it hadn't. (MCI's switching software was quite different +from AT&T's--though not necessarily any safer.) AT&T also announced +their plans to offer a rebate of service on Valentine's Day to make up +for the loss during the Crash. + +"Every technical resource available, including Bell Labs +scientists and engineers, has been devoted to assuring +it will not occur again," the public was told. They were +further assured that "The chances of a recurrence are small-- +a problem of this magnitude never occurred before." + +In the meantime, however, police and corporate +security maintained their own suspicions about +"the chances of recurrence" and the real reason why +a "problem of this magnitude" had appeared, seemingly +out of nowhere. Police and security knew for a fact +that hackers of unprecedented sophistication were illegally +entering, and reprogramming, certain digital switching stations. +Rumors of hidden "viruses" and secret "logic bombs" +in the switches ran rampant in the underground, +with much chortling over AT&T's predicament, +and idle speculation over what unsung hacker genius +was responsible for it. Some hackers, including police +informants, were trying hard to finger one another +as the true culprits of the Crash. + +Telco people found little comfort in objectivity when +they contemplated these possibilities. It was just too close +to the bone for them; it was embarrassing; it hurt so much, +it was hard even to talk about. + +There has always been thieving and misbehavior in the phone system. +There has always been trouble with the rival independents, +and in the local loops. But to have such trouble in the core +of the system, the long-distance switching stations, +is a horrifying affair. To telco people, this is +all the difference between finding roaches in your kitchen +and big horrid sewer-rats in your bedroom. + +From the outside, to the average citizen, the telcos +still seem gigantic and impersonal. The American public +seems to regard them as something akin to Soviet apparats. +Even when the telcos do their best corporate-citizen routine, +subsidizing magnet high-schools and sponsoring news-shows +on public television, they seem to win little except public suspicion. + +But from the inside, all this looks very different. +There's harsh competition. A legal and political system +that seems baffled and bored, when not actively hostile +to telco interests. There's a loss of morale, a deep sensation +of having somehow lost the upper hand. Technological change +has caused a loss of data and revenue to other, newer forms +of transmission. There's theft, and new forms of theft, +of growing scale and boldness and sophistication. +With all these factors, it was no surprise to see the telcos, +large and small, break out in a litany of bitter complaint. + +In late '88 and throughout 1989, telco representatives +grew shrill in their complaints to those few American law +enforcement officials who make it their business to try to +understand what telephone people are talking about. +Telco security officials had discovered the computer- +hacker underground, infiltrated it thoroughly, +and become deeply alarmed at its growing expertise. +Here they had found a target that was not only loathsome +on its face, but clearly ripe for counterattack. + +Those bitter rivals: AT&T, MCI and Sprint--and a crowd +of Baby Bells: PacBell, Bell South, Southwestern Bell, +NYNEX, USWest, as well as the Bell research consortium Bellcore, +and the independent long-distance carrier Mid-American-- +all were to have their role in the great hacker dragnet of 1990. +After years of being battered and pushed around, the telcos had, +at least in a small way, seized the initiative again. +After years of turmoil, telcos and government officials were +once again to work smoothly in concert in defense of the System. +Optimism blossomed; enthusiasm grew on all sides; +the prospective taste of vengeance was sweet. + +# + +From the beginning--even before the crackdown had a name-- +secrecy was a big problem. There were many good reasons +for secrecy in the hacker crackdown. Hackers and code-thieves +were wily prey, slinking back to their bedrooms and basements +and destroying vital incriminating evidence at the first hint of trouble. +Furthermore, the crimes themselves were heavily technical and difficult +to describe, even to police--much less to the general public. + +When such crimes HAD been described intelligibly to the public, +in the past, that very publicity had tended to INCREASE the crimes +enormously. Telco officials, while painfully aware of the vulnerabilities +of their systems, were anxious not to publicize those weaknesses. +Experience showed them that those weaknesses, once discovered, +would be pitilessly exploited by tens of thousands of people--not only +by professional grifters and by underground hackers and phone phreaks, +but by many otherwise more-or-less honest everyday folks, who regarded +stealing service from the faceless, soulless "Phone Company" as a kind of +harmless indoor sport. When it came to protecting their interests, +telcos had long since given up on general public sympathy for +"the Voice with a Smile." Nowadays the telco's "Voice" was +very likely to be a computer's; and the American public +showed much less of the proper respect and gratitude due +the fine public service bequeathed them by Dr. Bell and Mr. Vail. +The more efficient, high-tech, computerized, and impersonal +the telcos became, it seemed, the more they were met by +sullen public resentment and amoral greed. + +Telco officials wanted to punish the phone-phreak underground, in as +public and exemplary a manner as possible. They wanted to make dire +examples of the worst offenders, to seize the ringleaders and intimidate +the small fry, to discourage and frighten the wacky hobbyists, and send +the professional grifters to jail. To do all this, publicity was vital. + +Yet operational secrecy was even more so. If word got out that +a nationwide crackdown was coming, the hackers might simply vanish; +destroy the evidence, hide their computers, go to earth, +and wait for the campaign to blow over. Even the young +hackers were crafty and suspicious, and as for the professional grifters, +they tended to split for the nearest state-line at the first sign of trouble. +For the crackdown to work well, they would all have to be caught red-handed, +swept upon suddenly, out of the blue, from every corner of the compass. + +And there was another strong motive for secrecy. In the worst-case scenario, +a blown campaign might leave the telcos open to a devastating hacker +counter-attack. If there were indeed hackers loose in America who +had caused the January 15 Crash--if there were truly gifted hackers, +loose in the nation's long-distance switching systems, and enraged +or frightened by the crackdown--then they might react unpredictably +to an attempt to collar them. Even if caught, they might have talented +and vengeful friends still running around loose. Conceivably, +it could turn ugly. Very ugly. In fact, it was hard to imagine +just how ugly things might turn, given that possibility. + +Counter-attack from hackers was a genuine concern for the telcos. +In point of fact, they would never suffer any such counter-attack. +But in months to come, they would be at some pains to publicize +this notion and to utter grim warnings about it. + +Still, that risk seemed well worth running. Better to run the risk +of vengeful attacks, than to live at the mercy of potential crashers. +Any cop would tell you that a protection racket had no real future. + +And publicity was such a useful thing. Corporate security officers, +including telco security, generally work under conditions of great discretion. +And corporate security officials do not make money for their companies. +Their job is to PREVENT THE LOSS of money, which is much less glamorous +than actually winning profits. + +If you are a corporate security official, and you do your job brilliantly, +then nothing bad happens to your company at all. Because of this, you appear +completely superfluous. This is one of the many unattractive aspects +of security work. It's rare that these folks have the chance to draw +some healthy attention to their own efforts. + +Publicity also served the interest of their friends in law enforcement. +Public officials, including law enforcement officials, thrive by attracting +favorable public interest. A brilliant prosecution in a matter of vital +public interest can make the career of a prosecuting attorney. +And for a police officer, good publicity opens the purses of the legislature; +it may bring a citation, or a promotion, or at least a rise in status +and the respect of one's peers. + +But to have both publicity and secrecy is to have one's cake and eat it too. +In months to come, as we will show, this impossible act was to cause great +pain to the agents of the crackdown. But early on, it seemed possible +--maybe even likely--that the crackdown could successfully combine +the best of both worlds. The ARREST of hackers would be heavily publicized. +The actual DEEDS of the hackers, which were technically hard to explain +and also a security risk, would be left decently obscured. The THREAT +hackers posed would be heavily trumpeted; the likelihood of their actually +committing such fearsome crimes would be left to the public's imagination. +The spread of the computer underground, and its growing technical +sophistication, would be heavily promoted; the actual hackers themselves, +mostly bespectacled middle-class white suburban teenagers, +would be denied any personal publicity. + +It does not seem to have occurred to any telco official +that the hackers accused would demand a day in court; +that journalists would smile upon the hackers as +"good copy;" that wealthy high-tech entrepreneurs would +offer moral and financial support to crackdown victims; +that constitutional lawyers would show up with briefcases, +frowning mightily. This possibility does not seem to have +ever entered the game-plan. + +And even if it had, it probably would not have slowed +the ferocious pursuit of a stolen phone-company document, +mellifluously known as "Control Office Administration of +Enhanced 911 Services for Special Services and Major Account Centers." + +In the chapters to follow, we will explore the worlds +of police and the computer underground, and the large +shadowy area where they overlap. But first, we must +explore the battleground. Before we leave the world +of the telcos, we must understand what a switching system +actually is and how your telephone actually works. + +# + +To the average citizen, the idea of the telephone is represented by, +well, a TELEPHONE: a device that you talk into. To a telco +professional, however, the telephone itself is known, in lordly +fashion, as a "subset." The "subset" in your house is a mere adjunct, +a distant nerve ending, of the central switching stations, +which are ranked in levels of heirarchy, up to the long-distance electronic +switching stations, which are some of the largest computers on earth. + +Let us imagine that it is, say, 1925, before the +introduction of computers, when the phone system was +simpler and somewhat easier to grasp. Let's further +imagine that you are Miss Leticia Luthor, a fictional +operator for Ma Bell in New York City of the 20s. + +Basically, you, Miss Luthor, ARE the "switching system." +You are sitting in front of a large vertical switchboard, +known as a "cordboard," made of shiny wooden panels, +with ten thousand metal-rimmed holes punched in them, +known as jacks. The engineers would have put more +holes into your switchboard, but ten thousand is +as many as you can reach without actually having +to get up out of your chair. + +Each of these ten thousand holes has its own little electric lightbulb, +known as a "lamp," and its own neatly printed number code. + +With the ease of long habit, you are scanning your board for lit-up bulbs. +This is what you do most of the time, so you are used to it. + +A lamp lights up. This means that the phone +at the end of that line has been taken off the hook. +Whenever a handset is taken off the hook, that closes a circuit +inside the phone which then signals the local office, i.e. you, +automatically. There might be somebody calling, or then +again the phone might be simply off the hook, but this +does not matter to you yet. The first thing you do, +is record that number in your logbook, in your fine American +public-school handwriting. This comes first, naturally, +since it is done for billing purposes. + +You now take the plug of your answering cord, which goes +directly to your headset, and plug it into the lit-up hole. +"Operator," you announce. + +In operator's classes, before taking this job, you have +been issued a large pamphlet full of canned operator's +responses for all kinds of contingencies, which you had +to memorize. You have also been trained in a proper +non-regional, non-ethnic pronunciation and tone of voice. +You rarely have the occasion to make any spontaneous +remark to a customer, and in fact this is frowned upon +(except out on the rural lines where people have time +on their hands and get up to all kinds of mischief). + +A tough-sounding user's voice at the end of the line +gives you a number. Immediately, you write that number +down in your logbook, next to the caller's number, +which you just wrote earlier. You then look and see if +the number this guy wants is in fact on your switchboard, +which it generally is, since it's generally a local call. +Long distance costs so much that people use it sparingly. + +Only then do you pick up a calling-cord from a shelf +at the base of the switchboard. This is a long elastic cord +mounted on a kind of reel so that it will zip back in when +you unplug it. There are a lot of cords down there, +and when a bunch of them are out at once they look like +a nest of snakes. Some of the girls think there are bugs +living in those cable-holes. They're called "cable mites" +and are supposed to bite your hands and give you rashes. +You don't believe this, yourself. + +Gripping the head of your calling-cord, you slip the tip +of it deftly into the sleeve of the jack for the called person. +Not all the way in, though. You just touch it. If you hear +a clicking sound, that means the line is busy and you can't +put the call through. If the line is busy, you have to stick +the calling-cord into a "busy-tone jack," which will give +the guy a busy-tone. This way you don't have to talk to him +yourself and absorb his natural human frustration. + +But the line isn't busy. So you pop the cord all the way in. +Relay circuits in your board make the distant phone ring, +and if somebody picks it up off the hook, then a phone +conversation starts. You can hear this conversation +on your answering cord, until you unplug it. In fact +you could listen to the whole conversation if you wanted, +but this is sternly frowned upon by management, and frankly, +when you've overheard one, you've pretty much heard 'em all. + +You can tell how long the conversation lasts by the glow +of the calling-cord's lamp, down on the calling-cord's shelf. +When it's over, you unplug and the calling-cord zips back into place. + +Having done this stuff a few hundred thousand times, +you become quite good at it. In fact you're plugging, +and connecting, and disconnecting, ten, twenty, forty cords +at a time. It's a manual handicraft, really, quite satisfying +in a way, rather like weaving on an upright loom. + +Should a long-distance call come up, it would be different, +but not all that different. Instead of connecting the call +through your own local switchboard, you have to go up the hierarchy, +onto the long-distance lines, known as "trunklines." +Depending on how far the call goes, it may have to work +its way through a whole series of operators, which can +take quite a while. The caller doesn't wait on the line +while this complex process is negotiated across the country +by the gaggle of operators. Instead, the caller hangs up, +and you call him back yourself when the call has finally +worked its way through. + +After four or five years of this work, you get married, +and you have to quit your job, this being the natural order +of womanhood in the American 1920s. The phone company +has to train somebody else--maybe two people, since +the phone system has grown somewhat in the meantime. +And this costs money. + +In fact, to use any kind of human being as a switching +system is a very expensive proposition. Eight thousand +Leticia Luthors would be bad enough, but a quarter of a +million of them is a military-scale proposition and makes +drastic measures in automation financially worthwhile. + +Although the phone system continues to grow today, +the number of human beings employed by telcos has +been dropping steadily for years. Phone "operators" +now deal with nothing but unusual contingencies, +all routine operations having been shrugged off onto machines. +Consequently, telephone operators are considerably less +machine-like nowadays, and have been known to have accents +and actual character in their voices. When you reach +a human operator today, the operators are rather more +"human" than they were in Leticia's day--but on the other hand, +human beings in the phone system are much harder to reach +in the first place. + +Over the first half of the twentieth century, +"electromechanical" switching systems of growing +complexity were cautiously introduced into the phone system. +In certain backwaters, some of these hybrid systems are still +in use. But after 1965, the phone system began to go completely +electronic, and this is by far the dominant mode today. +Electromechanical systems have "crossbars," and "brushes," +and other large moving mechanical parts, which, while faster +and cheaper than Leticia, are still slow, and tend to wear out +fairly quickly. + +But fully electronic systems are inscribed on silicon chips, +and are lightning-fast, very cheap, and quite durable. +They are much cheaper to maintain than even the best +electromechanical systems, and they fit into half the space. +And with every year, the silicon chip grows smaller, faster, +and cheaper yet. Best of all, automated electronics work +around the clock and don't have salaries or health insurance. + +There are, however, quite serious drawbacks to the +use of computer-chips. When they do break down, it is +a daunting challenge to figure out what the heck has gone +wrong with them. A broken cordboard generally had +a problem in it big enough to see. A broken chip has +invisible, microscopic faults. And the faults in bad +software can be so subtle as to be practically theological. + +If you want a mechanical system to do something new, +then you must travel to where it is, and pull pieces out of it, +and wire in new pieces. This costs money. However, if you want +a chip to do something new, all you have to do is change its software, +which is easy, fast and dirt-cheap. You don't even have to see the chip +to change its program. Even if you did see the chip, it wouldn't look +like much. A chip with program X doesn't look one whit different from +a chip with program Y. + +With the proper codes and sequences, and access to specialized phone-lines, +you can change electronic switching systems all over America from anywhere +you please. + +And so can other people. If they know how, and if they want to, +they can sneak into a microchip via the special phonelines and diddle with it, +leaving no physical trace at all. If they broke into the operator's station +and held Leticia at gunpoint, that would be very obvious. If they broke into +a telco building and went after an electromechanical switch with a toolbelt, +that would at least leave many traces. But people can do all manner of amazing +things to computer switches just by typing on a keyboard, and keyboards are +everywhere today. The extent of this vulnerability is deep, dark, broad, +almost mind-boggling, and yet this is a basic, primal fact of life about +any computer on a network. + +Security experts over the past twenty years have insisted, +with growing urgency, that this basic vulnerability of computers +represents an entirely new level of risk, of unknown but obviously +dire potential to society. And they are right. + +An electronic switching station does pretty much +everything Letitia did, except in nanoseconds and +on a much larger scale. Compared to Miss Luthor's +ten thousand jacks, even a primitive 1ESS switching computer, +60s vintage, has a 128,000 lines. And the current AT&T +system of choice is the monstrous fifth-generation 5ESS. + +An Electronic Switching Station can scan every line on its "board" +in a tenth of a second, and it does this over and over, tirelessly, +around the clock. Instead of eyes, it uses "ferrod scanners" +to check the condition of local lines and trunks. Instead of hands, +it has "signal distributors," "central pulse distributors," +"magnetic latching relays," and "reed switches," which complete +and break the calls. Instead of a brain, it has a "central processor." +Instead of an instruction manual, it has a program. Instead of +a handwritten logbook for recording and billing calls, +it has magnetic tapes. And it never has to talk to anybody. +Everything a customer might say to it is done by punching +the direct-dial tone buttons on your subset. + +Although an Electronic Switching Station can't talk, +it does need an interface, some way to relate to its, er, +employers. This interface is known as the "master control +center." (This interface might be better known simply as +"the interface," since it doesn't actually "control" phone +calls directly. However, a term like "Master Control +Center" is just the kind of rhetoric that telco maintenance +engineers--and hackers--find particularly satisfying.) + +Using the master control center, a phone engineer can test +local and trunk lines for malfunctions. He (rarely she) +can check various alarm displays, measure traffic on the lines, +examine the records of telephone usage and the charges for those calls, +and change the programming. + +And, of course, anybody else who gets into the master control center +by remote control can also do these things, if he (rarely she) +has managed to figure them out, or, more likely, has somehow swiped +the knowledge from people who already know. + +In 1989 and 1990, one particular RBOC, BellSouth, +which felt particularly troubled, spent a purported $1.2 +million on computer security. Some think it spent as +much as two million, if you count all the associated costs. +Two million dollars is still very little compared to the +great cost-saving utility of telephonic computer systems. + +Unfortunately, computers are also stupid. +Unlike human beings, computers possess the truly +profound stupidity of the inanimate. + +In the 1960s, in the first shocks of spreading computerization, +there was much easy talk about the stupidity of computers-- +how they could "only follow the program" and were rigidly required +to do "only what they were told." There has been rather less talk +about the stupidity of computers since they began to achieve +grandmaster status in chess tournaments, and to manifest +many other impressive forms of apparent cleverness. + +Nevertheless, computers STILL are profoundly brittle and stupid; +they are simply vastly more subtle in their stupidity and brittleness. +The computers of the 1990s are much more reliable in their components +than earlier computer systems, but they are also called upon to do +far more complex things, under far more challenging conditions. + +On a basic mathematical level, every single line of +a software program offers a chance for some possible screwup. +Software does not sit still when it works; it "runs," +it interacts with itself and with its own inputs and outputs. +By analogy, it stretches like putty into millions of possible +shapes and conditions, so many shapes that they can never +all be successfully tested, not even in the lifespan of the universe. +Sometimes the putty snaps. + +The stuff we call "software" is not like anything that human society +is used to thinking about. Software is something like a machine, +and something like mathematics, and something like language, and +something like thought, and art, and information. . . . But software +is not in fact any of those other things. The protean quality +of software is one of the great sources of its fascination. +It also makes software very powerful, very subtle, +very unpredictable, and very risky. + +Some software is bad and buggy. Some is "robust," +even "bulletproof." The best software is that which has +been tested by thousands of users under thousands of +different conditions, over years. It is then known as +"stable." This does NOT mean that the software is +now flawless, free of bugs. It generally means that there +are plenty of bugs in it, but the bugs are well-identified +and fairly well understood. + +There is simply no way to assure that software is free +of flaws. Though software is mathematical in nature, +it cannot by "proven" like a mathematical theorem; +software is more like language, with inherent ambiguities, +with different definitions, different assumptions, +different levels of meaning that can conflict. + +Human beings can manage, more or less, with +human language because we can catch the gist of it. + +Computers, despite years of effort in "artificial intelligence," +have proven spectacularly bad in "catching the gist" of anything at all. +The tiniest bit of semantic grit may still bring the mightiest computer +tumbling down. One of the most hazardous things you can do to a +computer program is try to improve it--to try to make it safer. +Software "patches" represent new, untried un-"stable" software, +which is by definition riskier. + +The modern telephone system has come to depend, +utterly and irretrievably, upon software. And the +System Crash of January 15, 1990, was caused by an +IMPROVEMENT in software. Or rather, an ATTEMPTED +improvement. + +As it happened, the problem itself--the problem per se--took this form. +A piece of telco software had been written in C language, a standard +language of the telco field. Within the C software was a +long "do. . .while" construct. The "do. . .while" construct +contained a "switch" statement. The "switch" statement contained +an "if" clause. The "if" clause contained a "break." The "break" +was SUPPOSED to "break" the "if clause." Instead, the "break" +broke the "switch" statement. + +That was the problem, the actual reason why people picking up phones +on January 15, 1990, could not talk to one another. + +Or at least, that was the subtle, abstract, cyberspatial +seed of the problem. This is how the problem manifested itself +from the realm of programming into the realm of real life. + +The System 7 software for AT&T's 4ESS switching station, +the "Generic 44E14 Central Office Switch Software," +had been extensively tested, and was considered very stable. +By the end of 1989, eighty of AT&T's switching systems +nationwide had been programmed with the new software. Cautiously, +thirty-four stations were left to run the slower, less-capable +System 6, because AT&T suspected there might be shakedown problems +with the new and unprecedently sophisticated System 7 network. + +The stations with System 7 were programmed to switch over to a backup net +in case of any problems. In mid-December 1989, however, a new high-velocity, +high-security software patch was distributed to each of the 4ESS switches +that would enable them to switch over even more quickly, making the System 7 +network that much more secure. + +Unfortunately, every one of these 4ESS switches was now in possession +of a small but deadly flaw. + +In order to maintain the network, switches must monitor +the condition of other switches--whether they are up and running, +whether they have temporarily shut down, whether they are overloaded +and in need of assistance, and so forth. The new software helped +control this bookkeeping function by monitoring the status calls +from other switches. + +It only takes four to six seconds for a troubled 4ESS switch +to rid itself of all its calls, drop everything temporarily, +and re-boot its software from scratch. Starting over from scratch +will generally rid the switch of any software problems that may have +developed in the course of running the system. Bugs that arise will +be simply wiped out by this process. It is a clever idea. This process +of automatically re-booting from scratch is known as the "normal fault +recovery routine." Since AT&T's software is in fact exceptionally stable, +systems rarely have to go into "fault recovery" in the first place; +but AT&T has always boasted of its "real world" reliability, and this +tactic is a belt-and-suspenders routine. + +The 4ESS switch used its new software to monitor its fellow switches +as they recovered from faults. As other switches came back on line +after recovery, they would send their "OK" signals to the switch. +The switch would make a little note to that effect in its "status map," +recognizing that the fellow switch was back and ready to go, +and should be sent some calls and put back to regular work. + +Unfortunately, while it was busy bookkeeping with the status map, +the tiny flaw in the brand-new software came into play. +The flaw caused the 4ESS switch to interact, subtly but drastically, +with incoming telephone calls from human users. If--and only if-- +two incoming phone-calls happened to hit the switch within a hundredth +of a second, then a small patch of data would be garbled by the flaw. + +But the switch had been programmed to monitor itself +constantly for any possible damage to its data. +When the switch perceived that its data had been somehow garbled, +then it too would go down, for swift repairs to its software. +It would signal its fellow switches not to send any more work. +It would go into the fault-recovery mode for four to six seconds. +And then the switch would be fine again, and would send out its "OK, +ready for work" signal. + +However, the "OK, ready for work" signal was the VERY THING THAT +HAD CAUSED THE SWITCH TO GO DOWN IN THE FIRST PLACE. And ALL the +System 7 switches had the same flaw in their status-map software. +As soon as they stopped to make the bookkeeping note that their fellow +switch was "OK," then they too would become vulnerable to the slight +chance that two phone-calls would hit them within a hundredth of a second. + +At approximately 2:25 P.M. EST on Monday, January 15, +one of AT&T's 4ESS toll switching systems in New York City +had an actual, legitimate, minor problem. It went into fault +recovery routines, announced "I'm going down," then announced, +"I'm back, I'm OK." And this cheery message then blasted +throughout the network to many of its fellow 4ESS switches. + +Many of the switches, at first, completely escaped trouble. +These lucky switches were not hit by the coincidence of +two phone calls within a hundredth of a second. +Their software did not fail--at first. But three switches-- +in Atlanta, St. Louis, and Detroit--were unlucky, +and were caught with their hands full. And they went down. +And they came back up, almost immediately. And they too began +to broadcast the lethal message that they, too, were "OK" again, +activating the lurking software bug in yet other switches. + +As more and more switches did have that bit of bad luck +and collapsed, the call-traffic became more and more densely +packed in the remaining switches, which were groaning +to keep up with the load. And of course, as the calls +became more densely packed, the switches were MUCH MORE LIKELY +to be hit twice within a hundredth of a second. + +It only took four seconds for a switch to get well. +There was no PHYSICAL damage of any kind to the switches, +after all. Physically, they were working perfectly. +This situation was "only" a software problem. + +But the 4ESS switches were leaping up and down every +four to six seconds, in a virulent spreading wave all over America, +in utter, manic, mechanical stupidity. They kept KNOCKING +one another down with their contagious "OK" messages. + +It took about ten minutes for the chain reaction to cripple the network. +Even then, switches would periodically luck-out and manage to resume +their normal work. Many calls--millions of them--were managing +to get through. But millions weren't. + +The switching stations that used System 6 were not directly affected. +Thanks to these old-fashioned switches, AT&T's national system avoided +complete collapse. This fact also made it clear to engineers that +System 7 was at fault. + +Bell Labs engineers, working feverishly in New Jersey, Illinois, +and Ohio, first tried their entire repertoire of standard network +remedies on the malfunctioning System 7. None of the remedies worked, +of course, because nothing like this had ever happened to any +phone system before. + +By cutting out the backup safety network entirely, +they were able to reduce the frenzy of "OK" messages +by about half. The system then began to recover, as the +chain reaction slowed. By 11:30 P.M. on Monday January +15, sweating engineers on the midnight shift breathed a +sigh of relief as the last switch cleared-up. + +By Tuesday they were pulling all the brand-new 4ESS software +and replacing it with an earlier version of System 7. + +If these had been human operators, rather than +computers at work, someone would simply have +eventually stopped screaming. It would have been +OBVIOUS that the situation was not "OK," and common +sense would have kicked in. Humans possess common sense-- +at least to some extent. Computers simply don't. + +On the other hand, computers can handle hundreds +of calls per second. Humans simply can't. If every single +human being in America worked for the phone company, +we couldn't match the performance of digital switches: +direct-dialling, three-way calling, speed-calling, call- +waiting, Caller ID, all the rest of the cornucopia +of digital bounty. Replacing computers with operators +is simply not an option any more. + +And yet we still, anachronistically, expect humans to +be running our phone system. It is hard for us +to understand that we have sacrificed huge amounts +of initiative and control to senseless yet powerful machines. +When the phones fail, we want somebody to be responsible. +We want somebody to blame. + +When the Crash of January 15 happened, the American populace +was simply not prepared to understand that enormous landslides +in cyberspace, like the Crash itself, can happen, +and can be nobody's fault in particular. It was easier to believe, +maybe even in some odd way more reassuring to believe, +that some evil person, or evil group, had done this to us. +"Hackers" had done it. With a virus. A trojan horse. +A software bomb. A dirty plot of some kind. People believed this, +responsible people. In 1990, they were looking hard for evidence +to confirm their heartfelt suspicions. + +And they would look in a lot of places. + +Come 1991, however, the outlines of an apparent new reality +would begin to emerge from the fog. + +On July 1 and 2, 1991, computer-software collapses +in telephone switching stations disrupted service in +Washington DC, Pittsburgh, Los Angeles and San Francisco. +Once again, seemingly minor maintenance problems had +crippled the digital System 7. About twelve million +people were affected in the Crash of July 1, 1991. + +Said the New York Times Service: "Telephone company executives +and federal regulators said they were not ruling out the possibility +of sabotage by computer hackers, but most seemed to think the problems +stemmed from some unknown defect in the software running the networks." + +And sure enough, within the week, a red-faced software company, +DSC Communications Corporation of Plano, Texas, owned up +to "glitches" in the "signal transfer point" software that +DSC had designed for Bell Atlantic and Pacific Bell. +The immediate cause of the July 1 Crash was a single +mistyped character: one tiny typographical flaw +in one single line of the software. One mistyped letter, +in one single line, had deprived the nation's capital of phone service. +It was not particularly surprising that this tiny flaw had escaped attention: +a typical System 7 station requires TEN MILLION lines of code. + +On Tuesday, September 17, 1991, came the most spectacular outage yet. +This case had nothing to do with software failures--at least, not directly. +Instead, a group of AT&T's switching stations in New York City had simply +run out of electrical power and shut down cold. Their back-up batteries +had failed. Automatic warning systems were supposed to warn of the loss +of battery power, but those automatic systems had failed as well. + +This time, Kennedy, La Guardia, and Newark airports +all had their voice and data communications cut. +This horrifying event was particularly ironic, as attacks +on airport computers by hackers had long been a standard +nightmare scenario, much trumpeted by computer-security +experts who feared the computer underground. There had even +been a Hollywood thriller about sinister hackers ruining +airport computers--DIE HARD II. + +Now AT&T itself had crippled airports with computer malfunctions-- +not just one airport, but three at once, some of the busiest in the world. + +Air traffic came to a standstill throughout the Greater New York area, +causing more than 500 flights to be cancelled, in a spreading wave +all over America and even into Europe. Another 500 or so flights +were delayed, affecting, all in all, about 85,000 passengers. +(One of these passengers was the chairman of the Federal +Communications Commission.) + +Stranded passengers in New York and New Jersey were further +infuriated to discover that they could not even manage to +make a long distance phone call, to explain their delay +to loved ones or business associates. Thanks to the crash, +about four and a half million domestic calls, and half a million +international calls, failed to get through. + +The September 17 NYC Crash, unlike the previous ones, +involved not a whisper of "hacker" misdeeds. On the contrary, +by 1991, AT&T itself was suffering much of the vilification +that had formerly been directed at hackers. Congressmen were grumbling. +So were state and federal regulators. And so was the press. + +For their part, ancient rival MCI took out snide full-page +newspaper ads in New York, offering their own long-distance +services for the "next time that AT&T goes down." + +"You wouldn't find a classy company like AT&T using such advertising," +protested AT&T Chairman Robert Allen, unconvincingly. Once again, +out came the full-page AT&T apologies in newspapers, apologies for +"an inexcusable culmination of both human and mechanical failure." +(This time, however, AT&T offered no discount on later calls. +Unkind critics suggested that AT&T were worried about setting any precedent +for refunding the financial losses caused by telephone crashes.) + +Industry journals asked publicly if AT&T was "asleep at the switch." +The telephone network, America's purported marvel of high-tech reliability, +had gone down three times in 18 months. Fortune magazine listed the +Crash of September 17 among the "Biggest Business Goofs of 1991," +cruelly parodying AT&T's ad campaign in an article entitled +"AT&T Wants You Back (Safely On the Ground, God Willing)." + +Why had those New York switching systems simply run out of power? +Because no human being had attended to the alarm system. +Why did the alarm systems blare automatically, +without any human being noticing? Because the three +telco technicians who SHOULD have been listening +were absent from their stations in the power-room, +on another floor of the building--attending a training class. +A training class about the alarm systems for the power room! + +"Crashing the System" was no longer "unprecedented" by late 1991. +On the contrary, it no longer even seemed an oddity. By 1991, +it was clear that all the policemen in the world could no longer +"protect" the phone system from crashes. By far the worst crashes +the system had ever had, had been inflicted, by the system, +upon ITSELF. And this time nobody was making cocksure statements +that this was an anomaly, something that would never happen again. +By 1991 the System's defenders had met their nebulous Enemy, +and the Enemy was--the System. + + + +PART TWO: THE DIGITAL UNDERGROUND + + +The date was May 9, 1990. The Pope was touring Mexico City. +Hustlers from the Medellin Cartel were trying to buy +black-market Stinger missiles in Florida. On the comics page, +Doonesbury character Andy was dying of AIDS. And then. . .a highly +unusual item whose novelty and calculated rhetoric won it +headscratching attention in newspapers all over America. + +The US Attorney's office in Phoenix, Arizona, had issued +a press release announcing a nationwide law enforcement crackdown +against "illegal computer hacking activities." The sweep was +officially known as "Operation Sundevil." + +Eight paragraphs in the press release gave the bare facts: +twenty-seven search warrants carried out on May 8, with three arrests, +and a hundred and fifty agents on the prowl in "twelve" cities across America. +(Different counts in local press reports yielded "thirteen," "fourteen," and +"sixteen" cities.) Officials estimated that criminal losses of revenue +to telephone companies "may run into millions of dollars." Credit for +the Sundevil investigations was taken by the US Secret Service, +Assistant US Attorney Tim Holtzen of Phoenix, and the Assistant +Attorney General of Arizona, Gail Thackeray. + +The prepared remarks of Garry M. Jenkins, appearing in a U.S. Department +of Justice press release, were of particular interest. Mr. Jenkins was the +Assistant Director of the US Secret Service, and the highest-ranking federal +official to take any direct public role in the hacker crackdown of 1990. + +"Today, the Secret Service is sending a clear message to those computer hackers +who have decided to violate the laws of this nation in the mistaken belief +that they can successfully avoid detection by hiding behind the relative +anonymity of their computer terminals. (. . .) "Underground groups have been +formed for the purpose of exchanging information relevant to their criminal +activities. These groups often communicate with each other through message +systems between computers called `bulletin boards.' "Our experience shows +that many computer hacker suspects are no longer misguided teenagers, +mischievously playing games with their computers in their bedrooms. +Some are now high tech computer operators using computers to engage +in unlawful conduct." + +Who were these "underground groups" and "high-tech operators?" +Where had they come from? What did they want? Who WERE they? +Were they "mischievous?" Were they dangerous? How had "misguided teenagers" +managed to alarm the United States Secret Service? And just how widespread +was this sort of thing? + +Of all the major players in the Hacker Crackdown: the phone companies, +law enforcement, the civil libertarians, and the "hackers" themselves-- +the "hackers" are by far the most mysterious, by far the hardest to +understand, by far the WEIRDEST. + +Not only are "hackers" novel in their activities, but they come +in a variety of odd subcultures, with a variety of languages, +motives and values. + +The earliest proto-hackers were probably those unsung mischievous +telegraph boys who were summarily fired by the Bell Company in 1878. + +Legitimate "hackers," those computer enthusiasts who are independent-minded +but law-abiding, generally trace their spiritual ancestry to elite technical +universities, especially M.I.T. and Stanford, in the 1960s. + +But the genuine roots of the modern hacker UNDERGROUND can probably be traced +most successfully to a now much-obscured hippie anarchist movement known as +the Yippies. The Yippies, who took their name from the largely fictional +"Youth International Party," carried out a loud and lively policy of surrealistic +subversion and outrageous political mischief. Their basic tenets were flagrant +sexual promiscuity, open and copious drug use, the political overthrow of any +powermonger over thirty years of age, and an immediate end to the war +in Vietnam, by any means necessary, including the psychic levitation +of the Pentagon. + +The two most visible Yippies were Abbie Hoffman and Jerry Rubin. +Rubin eventually became a Wall Street broker. Hoffman, ardently sought +by federal authorities, went into hiding for seven years, +in Mexico, France, and the United States. While on the lam, +Hoffman continued to write and publish, with help from sympathizers +in the American anarcho-leftist underground. Mostly, Hoffman survived +through false ID and odd jobs. Eventually he underwent facial plastic +surgery and adopted an entirely new identity as one "Barry Freed." +After surrendering himself to authorities in 1980, Hoffman spent a year +in prison on a cocaine conviction. + +Hoffman's worldview grew much darker as the glory days of the 1960s faded. +In 1989, he purportedly committed suicide, under odd and, to some, rather +suspicious circumstances. + +Abbie Hoffman is said to have caused the Federal Bureau of Investigation +to amass the single largest investigation file ever opened on an individual +American citizen. (If this is true, it is still questionable whether the +FBI regarded Abbie Hoffman a serious public threat--quite possibly, +his file was enormous simply because Hoffman left colorful legendry +wherever he went). He was a gifted publicist, who regarded electronic +media as both playground and weapon. He actively enjoyed manipulating +network TV and other gullible, image-hungry media, with various weird lies, +mindboggling rumors, impersonation scams, and other sinister distortions, +all absolutely guaranteed to upset cops, Presidential candidates, +and federal judges. Hoffman's most famous work was a book self-reflexively +known as STEAL THIS BOOK, which publicized a number of methods by which young, +penniless hippie agitators might live off the fat of a system supported by +humorless drones. STEAL THIS BOOK, whose title urged readers to damage +the very means of distribution which had put it into their hands, +might be described as a spiritual ancestor of a computer virus. + +Hoffman, like many a later conspirator, made extensive use of +pay-phones for his agitation work--in his case, generally through +the use of cheap brass washers as coin-slugs. + +During the Vietnam War, there was a federal surtax imposed on telephone +service; Hoffman and his cohorts could, and did, argue that in systematically +stealing phone service they were engaging in civil disobedience: +virtuously denying tax funds to an illegal and immoral war. + +But this thin veil of decency was soon dropped entirely. +Ripping-off the System found its own justification in deep alienation +and a basic outlaw contempt for conventional bourgeois values. +Ingenious, vaguely politicized varieties of rip-off, +which might be described as "anarchy by convenience," +became very popular in Yippie circles, and because rip-off +was so useful, it was to survive the Yippie movement itself. + +In the early 1970s, it required fairly limited expertise +and ingenuity to cheat payphones, to divert "free" +electricity and gas service, or to rob vending machines +and parking meters for handy pocket change. It also required +a conspiracy to spread this knowledge, and the gall +and nerve actually to commit petty theft, but the Yippies +had these qualifications in plenty. In June 1971, Abbie +Hoffman and a telephone enthusiast sarcastically known +as "Al Bell" began publishing a newsletter called Youth +International Party Line. This newsletter was dedicated +to collating and spreading Yippie rip-off techniques, +especially of phones, to the joy of the freewheeling +underground and the insensate rage of all straight people. +As a political tactic, phone-service theft ensured +that Yippie advocates would always have ready access +to the long-distance telephone as a medium, despite +the Yippies' chronic lack of organization, discipline, +money, or even a steady home address. + +PARTY LINE was run out of Greenwich Village for a couple of years, +then "Al Bell" more or less defected from the faltering ranks of Yippiedom, +changing the newsletter's name to TAP or Technical Assistance Program. +After the Vietnam War ended, the steam began leaking rapidly out of American +radical dissent. But by this time, "Bell" and his dozen or so +core contributors had the bit between their teeth, +and had begun to derive tremendous gut-level satisfaction +from the sensation of pure TECHNICAL POWER. + +TAP articles, once highly politicized, became pitilessly jargonized +and technical, in homage or parody to the Bell System's own technical +documents, which TAP studied closely, gutted, and reproduced without +permission. The TAP elite revelled in gloating possession +of the specialized knowledge necessary to beat the system. + +"Al Bell" dropped out of the game by the late 70s, +and "Tom Edison" took over; TAP readers (some 1400 of +them, all told) now began to show more interest in telex +switches and the growing phenomenon of computer systems. + +In 1983, "Tom Edison" had his computer stolen and his house +set on fire by an arsonist. This was an eventually mortal blow +to TAP (though the legendary name was to be resurrected +in 1990 by a young Kentuckian computer-outlaw named "Predat0r.") + +# + +Ever since telephones began to make money, there have been +people willing to rob and defraud phone companies. +The legions of petty phone thieves vastly outnumber those +"phone phreaks" who "explore the system" for the sake +of the intellectual challenge. The New York metropolitan area +(long in the vanguard of American crime) claims over 150,000 +physical attacks on pay telephones every year! Studied carefully, +a modern payphone reveals itself as a little fortress, carefully +designed and redesigned over generations, to resist coin-slugs, +zaps of electricity, chunks of coin-shaped ice, prybars, magnets, +lockpicks, blasting caps. Public pay- phones must survive in a world +of unfriendly, greedy people, and a modern payphone is as exquisitely +evolved as a cactus. +Because the phone network pre-dates the computer network, +the scofflaws known as "phone phreaks" pre-date the scofflaws +known as "computer hackers." In practice, today, the line +between "phreaking" and "hacking" is very blurred, +just as the distinction between telephones and computers +has blurred. The phone system has been digitized, +and computers have learned to "talk" over phone-lines. +What's worse--and this was the point of the Mr. Jenkins +of the Secret Service--some hackers have learned to steal, +and some thieves have learned to hack. + +Despite the blurring, one can still draw a few useful +behavioral distinctions between "phreaks" and "hackers." +Hackers are intensely interested in the "system" per se, +and enjoy relating to machines. "Phreaks" are more +social, manipulating the system in a rough-and-ready +fashion in order to get through to other human beings, +fast, cheap and under the table. + +Phone phreaks love nothing so much as "bridges," +illegal conference calls of ten or twelve chatting +conspirators, seaboard to seaboard, lasting for many hours +--and running, of course, on somebody else's tab, +preferably a large corporation's. + +As phone-phreak conferences wear on, people drop out +(or simply leave the phone off the hook, while they +sashay off to work or school or babysitting), +and new people are phoned up and invited to join in, +from some other continent, if possible. Technical trivia, +boasts, brags, lies, head-trip deceptions, weird rumors, +and cruel gossip are all freely exchanged. + +The lowest rung of phone-phreaking is the theft of telephone access codes. +Charging a phone call to somebody else's stolen number is, of course, +a pig-easy way of stealing phone service, requiring practically no +technical expertise. This practice has been very widespread, +especially among lonely people without much money who are far from home. +Code theft has flourished especially in college dorms, military bases, +and, notoriously, among roadies for rock bands. Of late, code theft +has spread very rapidly among Third Worlders in the US, who pile up +enormous unpaid long-distance bills to the Caribbean, South America, +and Pakistan. + +The simplest way to steal phone-codes is simply to look over +a victim's shoulder as he punches-in his own code-number +on a public payphone. This technique is known as "shoulder-surfing," +and is especially common in airports, bus terminals, and train stations. +The code is then sold by the thief for a few dollars. The buyer abusing +the code has no computer expertise, but calls his Mom in New York, +Kingston or Caracas and runs up a huge bill with impunity. The losses +from this primitive phreaking activity are far, far greater than the +monetary losses caused by computer-intruding hackers. + +In the mid-to-late 1980s, until the introduction of sterner telco +security measures, COMPUTERIZED code theft worked like a charm, +and was virtually omnipresent throughout the digital underground, +among phreaks and hackers alike. This was accomplished through +programming one's computer to try random code numbers over the telephone +until one of them worked. Simple programs to do this were widely available +in the underground; a computer running all night was likely to come up with +a dozen or so useful hits. This could be repeated week after week until +one had a large library of stolen codes. + +Nowadays, the computerized dialling of hundreds of numbers +can be detected within hours and swiftly traced. +If a stolen code is repeatedly abused, this too can +be detected within a few hours. But for years in the 1980s, +the publication of stolen codes was a kind of elementary etiquette +for fledgling hackers. The simplest way to establish your bona-fides +as a raider was to steal a code through repeated random dialling +and offer it to the "community" for use. Codes could be both stolen, +and used, simply and easily from the safety of one's own bedroom, +with very little fear of detection or punishment. + +Before computers and their phone-line modems entered American homes +in gigantic numbers, phone phreaks had their own special telecommunications +hardware gadget, the famous "blue box." This fraud device (now rendered +increasingly useless by the digital evolution of the phone system) could +trick switching systems into granting free access to long-distance lines. +It did this by mimicking the system's own signal, a tone of 2600 hertz. + +Steven Jobs and Steve Wozniak, the founders of Apple Computer, Inc., +once dabbled in selling blue-boxes in college dorms in California. +For many, in the early days of phreaking, blue-boxing was scarcely +perceived as "theft," but rather as a fun (if sneaky) way to use +excess phone capacity harmlessly. After all, the long-distance +lines were JUST SITTING THERE. . . . Whom did it hurt, really? +If you're not DAMAGING the system, and you're not USING UP ANY +TANGIBLE RESOURCE, and if nobody FINDS OUT what you did, +then what real harm have you done? What exactly HAVE you "stolen," +anyway? If a tree falls in the forest and nobody hears it, +how much is the noise worth? Even now this remains a rather +dicey question. + +Blue-boxing was no joke to the phone companies, however. +Indeed, when Ramparts magazine, a radical publication in California, +printed the wiring schematics necessary to create a mute box in June 1972, +the magazine was seized by police and Pacific Bell phone-company officials. +The mute box, a blue-box variant, allowed its user to receive long-distance +calls free of charge to the caller. This device was closely described in a +Ramparts article wryly titled "Regulating the Phone Company In Your Home." +Publication of this article was held to be in violation of Californian +State Penal Code section 502.7, which outlaws ownership of wire-fraud +devices and the selling of "plans or instructions for any instrument, +apparatus, or device intended to avoid telephone toll charges." + +Issues of Ramparts were recalled or seized on the newsstands, +and the resultant loss of income helped put the magazine out of business. +This was an ominous precedent for free-expression issues, but the telco's +crushing of a radical-fringe magazine passed without serious challenge +at the time. Even in the freewheeling California 1970s, it was widely felt +that there was something sacrosanct about what the phone company knew; +that the telco had a legal and moral right to protect itself by shutting +off the flow of such illicit information. Most telco information was so +"specialized" that it would scarcely be understood by any honest member +of the public. If not published, it would not be missed. To print such +material did not seem part of the legitimate role of a free press. + +In 1990 there would be a similar telco-inspired attack +on the electronic phreak/hacking "magazine" Phrack. +The Phrack legal case became a central issue in the +Hacker Crackdown, and gave rise to great controversy. +Phrack would also be shut down, for a time, at least, +but this time both the telcos and their law-enforcement +allies would pay a much larger price for their actions. +The Phrack case will be examined in detail, later. + +Phone-phreaking as a social practice is still very +much alive at this moment. Today, phone-phreaking +is thriving much more vigorously than the better-known +and worse-feared practice of "computer hacking." +New forms of phreaking are spreading rapidly, following +new vulnerabilities in sophisticated phone services. + +Cellular phones are especially vulnerable; their chips +can be re-programmed to present a false caller ID +and avoid billing. Doing so also avoids police tapping, +making cellular-phone abuse a favorite among drug-dealers. +"Call-sell operations" using pirate cellular phones can, +and have, been run right out of the backs of cars, which move +from "cell" to "cell" in the local phone system, retailing +stolen long-distance service, like some kind of demented +electronic version of the neighborhood ice-cream truck. + +Private branch-exchange phone systems in large corporations +can be penetrated; phreaks dial-up a local company, enter its +internal phone-system, hack it, then use the company's own +PBX system to dial back out over the public network, +causing the company to be stuck with the resulting +long-distance bill. This technique is known as "diverting." +"Diverting" can be very costly, especially because phreaks +tend to travel in packs and never stop talking. +Perhaps the worst by-product of this "PBX fraud" +is that victim companies and telcos have sued one another +over the financial responsibility for the stolen calls, +thus enriching not only shabby phreaks but well-paid lawyers. + +"Voice-mail systems" can also be abused; phreaks +can seize their own sections of these sophisticated +electronic answering machines, and use them for trading +codes or knowledge of illegal techniques. Voice-mail +abuse does not hurt the company directly, but finding +supposedly empty slots in your company's answering +machine all crammed with phreaks eagerly chattering +and hey-duding one another in impenetrable jargon can +cause sensations of almost mystical repulsion and dread. + +Worse yet, phreaks have sometimes been known to react +truculently to attempts to "clean up" the voice-mail system. +Rather than humbly acquiescing to being thrown out of their playground, +they may very well call up the company officials at work (or at home) +and loudly demand free voice-mail addresses of their very own. +Such bullying is taken very seriously by spooked victims. + +Acts of phreak revenge against straight people are rare, +but voice-mail systems are especially tempting and vulnerable, +and an infestation of angry phreaks in one's voice-mail system is no joke. +They can erase legitimate messages; or spy on private messages; +or harass users with recorded taunts and obscenities. +They've even been known to seize control of voice-mail security, +and lock out legitimate users, or even shut down the system entirely. + +Cellular phone-calls, cordless phones, and ship-to-shore +telephony can all be monitored by various forms of radio; +this kind of "passive monitoring" is spreading explosively today. +Technically eavesdropping on other people's cordless and cellular +phone-calls is the fastest-growing area in phreaking today. +This practice strongly appeals to the lust for power and conveys +gratifying sensations of technical superiority over the eavesdropping +victim. Monitoring is rife with all manner of tempting evil mischief. +Simple prurient snooping is by far the most common activity. +But credit-card numbers unwarily spoken over the phone can be recorded, +stolen and used. And tapping people's phone-calls (whether through +active telephone taps or passive radio monitors) does lend itself +conveniently to activities like blackmail, industrial espionage, +and political dirty tricks. + +It should be repeated that telecommunications fraud, +the theft of phone service, causes vastly greater monetary +losses than the practice of entering into computers by stealth. +Hackers are mostly young suburban American white males, +and exist in their hundreds--but "phreaks" come from both sexes +and from many nationalities, ages and ethnic backgrounds, +and are flourishing in the thousands. + +# + +The term "hacker" has had an unfortunate history. +This book, The Hacker Crackdown, has little to say about +"hacking" in its finer, original sense. The term can signify +the free-wheeling intellectual exploration of the highest +and deepest potential of computer systems. Hacking can +describe the determination to make access to computers +and information as free and open as possible. Hacking +can involve the heartfelt conviction that beauty can +be found in computers, that the fine aesthetic in a perfect +program can liberate the mind and spirit. This is "hacking" +as it was defined in Steven Levy's much-praised history +of the pioneer computer milieu, Hackers, published in 1984. + +Hackers of all kinds are absolutely soaked through with heroic +anti-bureaucratic sentiment. Hackers long for recognition +as a praiseworthy cultural archetype, the postmodern electronic +equivalent of the cowboy and mountain man. Whether they deserve +such a reputation is something for history to decide. But many hackers-- +including those outlaw hackers who are computer intruders, and whose +activities are defined as criminal--actually attempt to LIVE UP TO +this techno-cowboy reputation. And given that electronics and +telecommunications are still largely unexplored territories, +there is simply NO TELLING what hackers might uncover. + +For some people, this freedom is the very breath of oxygen, +the inventive spontaneity that makes life worth living +and that flings open doors to marvellous possibility and +individual empowerment. But for many people +--and increasingly so--the hacker is an ominous figure, +a smart-aleck sociopath ready to burst out of his basement +wilderness and savage other people's lives for his own +anarchical convenience. + +Any form of power without responsibility, without direct +and formal checks and balances, is frightening to people-- +and reasonably so. It should be frankly admitted that +hackers ARE frightening, and that the basis of this fear +is not irrational. + +Fear of hackers goes well beyond the fear of merely criminal activity. + +Subversion and manipulation of the phone system +is an act with disturbing political overtones. +In America, computers and telephones are potent symbols +of organized authority and the technocratic business elite. + +But there is an element in American culture that +has always strongly rebelled against these symbols; +rebelled against all large industrial computers +and all phone companies. A certain anarchical tinge deep +in the American soul delights in causing confusion and pain +to all bureaucracies, including technological ones. + +There is sometimes malice and vandalism in this attitude, +but it is a deep and cherished part of the American national character. +The outlaw, the rebel, the rugged individual, the pioneer, +the sturdy Jeffersonian yeoman, the private citizen resisting +interference in his pursuit of happiness--these are figures that all +Americans recognize, and that many will strongly applaud and defend. + +Many scrupulously law-abiding citizens today do cutting-edge work +with electronics--work that has already had tremendous social influence +and will have much more in years to come. In all truth, these talented, +hardworking, law-abiding, mature, adult people are far more disturbing +to the peace and order of the current status quo than any scofflaw group +of romantic teenage punk kids. These law-abiding hackers have the power, +ability, and willingness to influence other people's lives quite unpredictably. +They have means, motive, and opportunity to meddle drastically with the +American social order. When corralled into governments, universities, +or large multinational companies, and forced to follow rulebooks +and wear suits and ties, they at least have some conventional halters +on their freedom of action. But when loosed alone, or in small groups, +and fired by imagination and the entrepreneurial spirit, they can move +mountains--causing landslides that will likely crash directly into your +office and living room. + +These people, as a class, instinctively recognize that a public, +politicized attack on hackers will eventually spread to them-- +that the term "hacker," once demonized, might be used to knock +their hands off the levers of power and choke them out of existence. +There are hackers today who fiercely and publicly resist any besmirching +of the noble title of hacker. Naturally and understandably, they deeply +resent the attack on their values implicit in using the word "hacker" +as a synonym for computer-criminal. + +This book, sadly but in my opinion unavoidably, rather adds +to the degradation of the term. It concerns itself mostly with "hacking" +in its commonest latter-day definition, i.e., intruding into computer +systems by stealth and without permission. The term "hacking" is used +routinely today by almost all law enforcement officials with any +professional interest in computer fraud and abuse. American police +describe almost any crime committed with, by, through, or against +a computer as hacking. + +Most importantly, "hacker" is what computer-intruders +choose to call THEMSELVES. Nobody who "hacks" into systems +willingly describes himself (rarely, herself) as a "computer intruder," +"computer trespasser," "cracker," "wormer," "darkside hacker" +or "high tech street gangster." Several other demeaning terms +have been invented in the hope that the press and public +will leave the original sense of the word alone. But few people +actually use these terms. (I exempt the term "cyberpunk," +which a few hackers and law enforcement people actually do use. +The term "cyberpunk" is drawn from literary criticism and has +some odd and unlikely resonances, but, like hacker, +cyberpunk too has become a criminal pejorative today.) + +In any case, breaking into computer systems was hardly alien +to the original hacker tradition. The first tottering systems +of the 1960s required fairly extensive internal surgery merely +to function day-by-day. Their users "invaded" the deepest, +most arcane recesses of their operating software almost +as a matter of routine. "Computer security" in these early, +primitive systems was at best an afterthought. What security +there was, was entirely physical, for it was assumed that +anyone allowed near this expensive, arcane hardware would be +a fully qualified professional expert. + +In a campus environment, though, this meant that grad students, +teaching assistants, undergraduates, and eventually, +all manner of dropouts and hangers-on ended up accessing +and often running the works. + +Universities, even modern universities, are not in +the business of maintaining security over information. +On the contrary, universities, as institutions, pre-date +the "information economy" by many centuries and are not- +for-profit cultural entities, whose reason for existence +(purportedly) is to discover truth, codify it through +techniques of scholarship, and then teach it. Universities +are meant to PASS THE TORCH OF CIVILIZATION, not just +download data into student skulls, and the values of the +academic community are strongly at odds with those of all +would-be information empires. Teachers at all levels, from +kindergarten up, have proven to be shameless and persistent +software and data pirates. Universities do not merely +"leak information" but vigorously broadcast free thought. + +This clash of values has been fraught with controversy. +Many hackers of the 1960s remember their professional +apprenticeship as a long guerilla war against the uptight +mainframe-computer "information priesthood." These computer-hungry +youngsters had to struggle hard for access to computing power, +and many of them were not above certain, er, shortcuts. +But, over the years, this practice freed computing +from the sterile reserve of lab-coated technocrats and +was largely responsible for the explosive growth of computing +in general society--especially PERSONAL computing. + +Access to technical power acted like catnip on certain +of these youngsters. Most of the basic techniques of +computer intrusion: password cracking, trapdoors, backdoors, +trojan horses--were invented in college environments in the 1960s, +in the early days of network computing. Some off-the-cuff +experience at computer intrusion was to be in the informal +resume of most "hackers" and many future industry giants. +Outside of the tiny cult of computer enthusiasts, few people +thought much about the implications of "breaking into" +computers. This sort of activity had not yet been publicized, +much less criminalized. + +In the 1960s, definitions of "property" and "privacy" +had not yet been extended to cyberspace. Computers +were not yet indispensable to society. There were no vast +databanks of vulnerable, proprietary information stored +in computers, which might be accessed, copied without +permission, erased, altered, or sabotaged. The stakes +were low in the early days--but they grew every year, +exponentially, as computers themselves grew. + +By the 1990s, commercial and political pressures +had become overwhelming, and they broke the social +boundaries of the hacking subculture. Hacking +had become too important to be left to the hackers. +Society was now forced to tackle the intangible nature +of cyberspace-as-property, cyberspace as privately-owned +unreal-estate. In the new, severe, responsible, high-stakes +context of the "Information Society" of the 1990s, +"hacking" was called into question. + +What did it mean to break into a computer without +permission and use its computational power, or look +around inside its files without hurting anything? +What were computer-intruding hackers, anyway--how should +society, and the law, best define their actions? +Were they just BROWSERS, harmless intellectual explorers? +Were they VOYEURS, snoops, invaders of privacy? Should +they be sternly treated as potential AGENTS OF ESPIONAGE, +or perhaps as INDUSTRIAL SPIES? Or were they best +defined as TRESPASSERS, a very common teenage +misdemeanor? Was hacking THEFT OF SERVICE? +(After all, intruders were getting someone else's +computer to carry out their orders, without permission +and without paying). Was hacking FRAUD? Maybe it was +best described as IMPERSONATION. The commonest mode +of computer intrusion was (and is) to swipe or snoop +somebody else's password, and then enter the computer +in the guise of another person--who is commonly stuck +with the blame and the bills. + +Perhaps a medical metaphor was better--hackers should +be defined as "sick," as COMPUTER ADDICTS unable +to control their irresponsible, compulsive behavior. + +But these weighty assessments meant little to the +people who were actually being judged. From inside +the underground world of hacking itself, all these +perceptions seem quaint, wrongheaded, stupid, or meaningless. +The most important self-perception of underground hackers-- +from the 1960s, right through to the present day--is that +they are an ELITE. The day-to-day struggle in the underground +is not over sociological definitions--who cares?--but for power, +knowledge, and status among one's peers. + +When you are a hacker, it is your own inner conviction +of your elite status that enables you to break, or let +us say "transcend," the rules. It is not that ALL rules +go by the board. The rules habitually broken by hackers +are UNIMPORTANT rules--the rules of dopey greedhead telco +bureaucrats and pig-ignorant government pests. + +Hackers have their OWN rules, which separate behavior +which is cool and elite, from behavior which is rodentlike, +stupid and losing. These "rules," however, are mostly unwritten +and enforced by peer pressure and tribal feeling. Like all rules +that depend on the unspoken conviction that everybody else +is a good old boy, these rules are ripe for abuse. The mechanisms +of hacker peer- pressure, "teletrials" and ostracism, are rarely used +and rarely work. Back-stabbing slander, threats, and electronic +harassment are also freely employed in down-and-dirty intrahacker feuds, +but this rarely forces a rival out of the scene entirely. The only real +solution for the problem of an utterly losing, treacherous and rodentlike +hacker is to TURN HIM IN TO THE POLICE. Unlike the Mafia or Medellin Cartel, +the hacker elite cannot simply execute the bigmouths, creeps and troublemakers +among their ranks, so they turn one another in with astonishing frequency. + +There is no tradition of silence or OMERTA in the hacker underworld. +Hackers can be shy, even reclusive, but when they do talk, hackers +tend to brag, boast and strut. Almost everything hackers do is INVISIBLE; +if they don't brag, boast, and strut about it, then NOBODY WILL EVER KNOW. +If you don't have something to brag, boast, and strut about, then nobody +in the underground will recognize you and favor you with vital cooperation +and respect. + +The way to win a solid reputation in the underground +is by telling other hackers things that could only +have been learned by exceptional cunning and stealth. +Forbidden knowledge, therefore, is the basic currency +of the digital underground, like seashells among +Trobriand Islanders. Hackers hoard this knowledge, +and dwell upon it obsessively, and refine it, +and bargain with it, and talk and talk about it. + +Many hackers even suffer from a strange obsession to TEACH-- +to spread the ethos and the knowledge of the digital underground. +They'll do this even when it gains them no particular advantage +and presents a grave personal risk. + +And when that risk catches up with them, they will go right on teaching +and preaching--to a new audience this time, their interrogators from law +enforcement. Almost every hacker arrested tells everything he knows-- +all about his friends, his mentors, his disciples--legends, threats, +horror stories, dire rumors, gossip, hallucinations. This is, of course, +convenient for law enforcement--except when law enforcement begins +to believe hacker legendry. + +Phone phreaks are unique among criminals in their willingness +to call up law enforcement officials--in the office, at their homes-- +and give them an extended piece of their mind. It is hard not to +interpret this as BEGGING FOR ARREST, and in fact it is an act +of incredible foolhardiness. Police are naturally nettled +by these acts of chutzpah and will go well out of their way +to bust these flaunting idiots. But it can also be interpreted +as a product of a world-view so elitist, so closed and hermetic, +that electronic police are simply not perceived as "police," +but rather as ENEMY PHONE PHREAKS who should be scolded +into behaving "decently." + +Hackers at their most grandiloquent perceive themselves +as the elite pioneers of a new electronic world. +Attempts to make them obey the democratically +established laws of contemporary American society are +seen as repression and persecution. After all, they argue, +if Alexander Graham Bell had gone along with the rules +of the Western Union telegraph company, there would have +been no telephones. If Jobs and Wozniak had believed +that IBM was the be-all and end-all, there would have +been no personal computers. If Benjamin Franklin and +Thomas Jefferson had tried to "work within the system" +there would have been no United States. + +Not only do hackers privately believe this as an article of faith, +but they have been known to write ardent manifestos about it. +Here are some revealing excerpts from an especially vivid hacker manifesto: +"The Techno-Revolution" by "Dr. Crash," which appeared in electronic +form in Phrack Volume 1, Issue 6, Phile 3. + + +"To fully explain the true motives behind hacking, +we must first take a quick look into the past. In the 1960s, +a group of MIT students built the first modern computer system. +This wild, rebellious group of young men were the first to bear +the name `hackers.' The systems that they developed were intended +to be used to solve world problems and to benefit all of mankind. +"As we can see, this has not been the case. The computer system +has been solely in the hands of big businesses and the government. +The wonderful device meant to enrich life has become a weapon which +dehumanizes people. To the government and large businesses, +people are no more than disk space, and the government doesn't +use computers to arrange aid for the poor, but to control nuclear +death weapons. The average American can only have access +to a small microcomputer which is worth only a fraction +of what they pay for it. The businesses keep the +true state-of-the-art equipment away from the people +behind a steel wall of incredibly high prices and bureaucracy. +It is because of this state of affairs that hacking was born. (. . .) +"Of course, the government doesn't want the monopoly of technology broken, +so they have outlawed hacking and arrest anyone who is caught. (. . .) +The phone company is another example of technology abused and kept +from people with high prices. (. . .) "Hackers often find that their +existing equipment, due to the monopoly tactics of computer companies, +is inefficient for their purposes. Due to the exorbitantly high prices, +it is impossible to legally purchase the necessary equipment. +This need has given still another segment of the fight: Credit Carding. +Carding is a way of obtaining the necessary goods without paying for them. +It is again due to the companies' stupidity that Carding is so easy, +and shows that the world's businesses are in the hands of those +with considerably less technical know-how than we, the hackers. (. . .) +"Hacking must continue. We must train newcomers to the art of hacking. +(. . . .) And whatever you do, continue the fight. Whether you know it +or not, if you are a hacker, you are a revolutionary. Don't worry, +you're on the right side." + +The defense of "carding" is rare. Most hackers regard credit-card +theft as "poison" to the underground, a sleazy and immoral effort that, +worse yet, is hard to get away with. Nevertheless, manifestos advocating +credit-card theft, the deliberate crashing of computer systems, +and even acts of violent physical destruction such as vandalism +and arson do exist in the underground. These boasts and threats +are taken quite seriously by the police. And not every hacker +is an abstract, Platonic computer-nerd. Some few are quite experienced +at picking locks, robbing phone-trucks, and breaking and entering buildings. + +Hackers vary in their degree of hatred for authority +and the violence of their rhetoric. But, at a bottom line, +they are scofflaws. They don't regard the current rules +of electronic behavior as respectable efforts to preserve +law and order and protect public safety. They regard these +laws as immoral efforts by soulless corporations to protect +their profit margins and to crush dissidents. "Stupid" people, +including police, businessmen, politicians, and journalists, +simply have no right to judge the actions of those possessed of genius, +techno-revolutionary intentions, and technical expertise. + +# + +Hackers are generally teenagers and college kids not +engaged in earning a living. They often come from fairly +well-to-do middle-class backgrounds, and are markedly +anti-materialistic (except, that is, when it comes to +computer equipment). Anyone motivated by greed for +mere money (as opposed to the greed for power, +knowledge and status) is swiftly written-off as a narrow- +minded breadhead whose interests can only be corrupt +and contemptible. Having grown up in the 1970s and +1980s, the young Bohemians of the digital underground +regard straight society as awash in plutocratic corruption, +where everyone from the President down is for sale and +whoever has the gold makes the rules. + +Interestingly, there's a funhouse-mirror image of this attitude +on the other side of the conflict. The police are also +one of the most markedly anti-materialistic groups +in American society, motivated not by mere money +but by ideals of service, justice, esprit-de-corps, +and, of course, their own brand of specialized knowledge +and power. Remarkably, the propaganda war between cops +and hackers has always involved angry allegations +that the other side is trying to make a sleazy buck. +Hackers consistently sneer that anti-phreak prosecutors +are angling for cushy jobs as telco lawyers and that +computer-crime police are aiming to cash in later +as well-paid computer-security consultants in the private sector. + +For their part, police publicly conflate all +hacking crimes with robbing payphones with crowbars. +Allegations of "monetary losses" from computer intrusion +are notoriously inflated. The act of illicitly copying +a document from a computer is morally equated with +directly robbing a company of, say, half a million dollars. +The teenage computer intruder in possession of this "proprietary" +document has certainly not sold it for such a sum, would likely +have little idea how to sell it at all, and quite probably +doesn't even understand what he has. He has not made a cent +in profit from his felony but is still morally equated with +a thief who has robbed the church poorbox and lit out for Brazil. + +Police want to believe that all hackers are thieves. +It is a tortuous and almost unbearable act for the American +justice system to put people in jail because they want +to learn things which are forbidden for them to know. +In an American context, almost any pretext for punishment +is better than jailing people to protect certain restricted +kinds of information. Nevertheless, POLICING INFORMATION +is part and parcel of the struggle against hackers. + +This dilemma is well exemplified by the remarkable +activities of "Emmanuel Goldstein," editor and publisher +of a print magazine known as 2600: The Hacker Quarterly. +Goldstein was an English major at Long Island's State University +of New York in the '70s, when he became involved with the local +college radio station. His growing interest in electronics +caused him to drift into Yippie TAP circles and thus into +the digital underground, where he became a self-described +techno-rat. His magazine publishes techniques of computer +intrusion and telephone "exploration" as well as gloating +exposes of telco misdeeds and governmental failings. + +Goldstein lives quietly and very privately in a large, +crumbling Victorian mansion in Setauket, New York. +The seaside house is decorated with telco decals, chunks of +driftwood, and the basic bric-a-brac of a hippie crash-pad. +He is unmarried, mildly unkempt, and survives mostly +on TV dinners and turkey-stuffing eaten straight out +of the bag. Goldstein is a man of considerable charm +and fluency, with a brief, disarming smile and the kind +of pitiless, stubborn, thoroughly recidivist integrity +that America's electronic police find genuinely alarming. + +Goldstein took his nom-de-plume, or "handle," from +a character in Orwell's 1984, which may be taken, +correctly, as a symptom of the gravity of his sociopolitical +worldview. He is not himself a practicing computer +intruder, though he vigorously abets these actions, +especially when they are pursued against large +corporations or governmental agencies. Nor is he a thief, +for he loudly scorns mere theft of phone service, in favor +of "exploring and manipulating the system." He is probably +best described and understood as a DISSIDENT. + +Weirdly, Goldstein is living in modern America +under conditions very similar to those of former +East European intellectual dissidents. In other words, +he flagrantly espouses a value-system that is deeply +and irrevocably opposed to the system of those in power +and the police. The values in 2600 are generally expressed +in terms that are ironic, sarcastic, paradoxical, or just +downright confused. But there's no mistaking their +radically anti-authoritarian tenor. 2600 holds that +technical power and specialized knowledge, of any kind +obtainable, belong by right in the hands of those individuals +brave and bold enough to discover them--by whatever means necessary. +Devices, laws, or systems that forbid access, and the free +spread of knowledge, are provocations that any free +and self-respecting hacker should relentlessly attack. +The "privacy" of governments, corporations and other soulless +technocratic organizations should never be protected +at the expense of the liberty and free initiative +of the individual techno-rat. + +However, in our contemporary workaday world, both governments +and corporations are very anxious indeed to police information +which is secret, proprietary, restricted, confidential, +copyrighted, patented, hazardous, illegal, unethical, +embarrassing, or otherwise sensitive. This makes Goldstein +persona non grata, and his philosophy a threat. + +Very little about the conditions of Goldstein's daily +life would astonish, say, Vaclav Havel. (We may note +in passing that President Havel once had his word-processor +confiscated by the Czechoslovak police.) Goldstein lives +by SAMIZDAT, acting semi-openly as a data-center +for the underground, while challenging the powers-that-be +to abide by their own stated rules: freedom of speech +and the First Amendment. + +Goldstein thoroughly looks and acts the part of techno-rat, +with shoulder-length ringlets and a piratical black +fisherman's-cap set at a rakish angle. He often shows up +like Banquo's ghost at meetings of computer professionals, +where he listens quietly, half-smiling and taking thorough notes. + +Computer professionals generally meet publicly, +and find it very difficult to rid themselves of Goldstein +and his ilk without extralegal and unconstitutional actions. +Sympathizers, many of them quite respectable people +with responsible jobs, admire Goldstein's attitude and +surreptitiously pass him information. An unknown but +presumably large proportion of Goldstein's 2,000-plus +readership are telco security personnel and police, +who are forced to subscribe to 2600 to stay abreast +of new developments in hacking. They thus find themselves +PAYING THIS GUY'S RENT while grinding their teeth in anguish, +a situation that would have delighted Abbie Hoffman +(one of Goldstein's few idols). + +Goldstein is probably the best-known public representative +of the hacker underground today, and certainly the best-hated. +Police regard him as a Fagin, a corrupter of youth, and speak +of him with untempered loathing. He is quite an accomplished gadfly. +After the Martin Luther King Day Crash of 1990, Goldstein, +for instance, adeptly rubbed salt into the wound in the pages of 2600. +"Yeah, it was fun for the phone phreaks as we watched the network crumble," +he admitted cheerfully. "But it was also an ominous sign of what's +to come. . . . Some AT&T people, aided by well-meaning but ignorant media, +were spreading the notion that many companies had the same software +and therefore could face the same problem someday. Wrong. This was +entirely an AT&T software deficiency. Of course, other companies could +face entirely DIFFERENT software problems. But then, so too could AT&T." + +After a technical discussion of the system's failings, +the Long Island techno-rat went on to offer thoughtful +criticism to the gigantic multinational's hundreds of +professionally qualified engineers. "What we don't know +is how a major force in communications like AT&T could +be so sloppy. What happened to backups? Sure, +computer systems go down all the time, but people +making phone calls are not the same as people logging +on to computers. We must make that distinction. It's not +acceptable for the phone system or any other essential +service to `go down.' If we continue to trust technology +without understanding it, we can look forward to many +variations on this theme. + +"AT&T owes it to its customers to be prepared to INSTANTLY +switch to another network if something strange and unpredictable +starts occurring. The news here isn't so much the failure +of a computer program, but the failure of AT&T's entire structure." + +The very idea of this. . . . this PERSON. . . . offering +"advice" about "AT&T's entire structure" is more than +some people can easily bear. How dare this near-criminal +dictate what is or isn't "acceptable" behavior from AT&T? +Especially when he's publishing, in the very same issue, +detailed schematic diagrams for creating various switching-network +signalling tones unavailable to the public. + +"See what happens when you drop a `silver box' tone or two +down your local exchange or through different long distance +service carriers," advises 2600 contributor "Mr. Upsetter" +in "How To Build a Signal Box." "If you experiment systematically +and keep good records, you will surely discover something interesting." + +This is, of course, the scientific method, generally regarded +as a praiseworthy activity and one of the flowers of modern civilization. +One can indeed learn a great deal with this sort of structured +intellectual activity. Telco employees regard this mode of "exploration" +as akin to flinging sticks of dynamite into their pond to see what lives +on the bottom. + +2600 has been published consistently since 1984. +It has also run a bulletin board computer system, +printed 2600 T-shirts, taken fax calls. . . . +The Spring 1991 issue has an interesting announcement on page 45: +"We just discovered an extra set of wires attached to our fax line +and heading up the pole. (They've since been clipped.) +Your faxes to us and to anyone else could be monitored." +In the worldview of 2600, the tiny band of techno-rat brothers +(rarely, sisters) are a beseiged vanguard of the truly free and honest. +The rest of the world is a maelstrom of corporate crime and high-level +governmental corruption, occasionally tempered with well-meaning +ignorance. To read a few issues in a row is to enter a nightmare +akin to Solzhenitsyn's, somewhat tempered by the fact that 2600 +is often extremely funny. + +Goldstein did not become a target of the Hacker Crackdown, +though he protested loudly, eloquently, and publicly about it, +and it added considerably to his fame. It was not that he is not +regarded as dangerous, because he is so regarded. Goldstein has had +brushes with the law in the past: in 1985, a 2600 bulletin board +computer was seized by the FBI, and some software on it was formally +declared "a burglary tool in the form of a computer program." +But Goldstein escaped direct repression in 1990, because his +magazine is printed on paper, and recognized as subject +to Constitutional freedom of the press protection. +As was seen in the Ramparts case, this is far from +an absolute guarantee. Still, as a practical matter, +shutting down 2600 by court-order would create so much +legal hassle that it is simply unfeasible, at least +for the present. Throughout 1990, both Goldstein +and his magazine were peevishly thriving. + +Instead, the Crackdown of 1990 would concern itself +with the computerized version of forbidden data. +The crackdown itself, first and foremost, was about +BULLETIN BOARD SYSTEMS. Bulletin Board Systems, most often +known by the ugly and un-pluralizable acronym "BBS," are +the life-blood of the digital underground. Boards were +also central to law enforcement's tactics and strategy +in the Hacker Crackdown. + +A "bulletin board system" can be formally defined as +a computer which serves as an information and message- +passing center for users dialing-up over the phone-lines +through the use of modems. A "modem," or modulator- +demodulator, is a device which translates the digital +impulses of computers into audible analog telephone +signals, and vice versa. Modems connect computers +to phones and thus to each other. + +Large-scale mainframe computers have been connected since the 1960s, +but PERSONAL computers, run by individuals out of their homes, +were first networked in the late 1970s. The "board" created +by Ward Christensen and Randy Suess in February 1978, +in Chicago, Illinois, is generally regarded as the first +personal-computer bulletin board system worthy of the name. + +Boards run on many different machines, employing many +different kinds of software. Early boards were crude and buggy, +and their managers, known as "system operators" or "sysops," +were hard-working technical experts who wrote their own software. +But like most everything else in the world of electronics, +boards became faster, cheaper, better-designed, and generally +far more sophisticated throughout the 1980s. They also moved +swiftly out of the hands of pioneers and into those of the +general public. By 1985 there were something in the +neighborhood of 4,000 boards in America. By 1990 it was +calculated, vaguely, that there were about 30,000 boards in +the US, with uncounted thousands overseas. + +Computer bulletin boards are unregulated enterprises. +Running a board is a rough-and-ready, catch-as-catch-can proposition. +Basically, anybody with a computer, modem, software and a phone-line +can start a board. With second-hand equipment and public-domain +free software, the price of a board might be quite small-- +less than it would take to publish a magazine or even a +decent pamphlet. Entrepreneurs eagerly sell bulletin-board +software, and will coach nontechnical amateur sysops in its use. + +Boards are not "presses." They are not magazines, +or libraries, or phones, or CB radios, or traditional cork +bulletin boards down at the local laundry, though they +have some passing resemblance to those earlier media. +Boards are a new medium--they may even be a LARGE NUMBER of new media. + +Consider these unique characteristics: boards are cheap, +yet they can have a national, even global reach. +Boards can be contacted from anywhere in the global +telephone network, at NO COST to the person running the board-- +the caller pays the phone bill, and if the caller is local, +the call is free. Boards do not involve an editorial elite +addressing a mass audience. The "sysop" of a board is not +an exclusive publisher or writer--he is managing an electronic salon, +where individuals can address the general public, play the part +of the general public, and also exchange private mail +with other individuals. And the "conversation" on boards, +though fluid, rapid, and highly interactive, is not spoken, +but written. It is also relatively anonymous, sometimes completely so. + +And because boards are cheap and ubiquitous, regulations +and licensing requirements would likely be practically unenforceable. +It would almost be easier to "regulate," "inspect," and "license" +the content of private mail--probably more so, since the mail system +is operated by the federal government. Boards are run by individuals, +independently, entirely at their own whim. + +For the sysop, the cost of operation is not the primary +limiting factor. Once the investment in a computer and +modem has been made, the only steady cost is the charge +for maintaining a phone line (or several phone lines). +The primary limits for sysops are time and energy. +Boards require upkeep. New users are generally "validated"-- +they must be issued individual passwords, and called at +home by voice-phone, so that their identity can be +verified. Obnoxious users, who exist in plenty, must be +chided or purged. Proliferating messages must be deleted +when they grow old, so that the capacity of the system +is not overwhelmed. And software programs (if such things +are kept on the board) must be examined for possible +computer viruses. If there is a financial charge to use +the board (increasingly common, especially in larger and +fancier systems) then accounts must be kept, and users +must be billed. And if the board crashes--a very common +occurrence--then repairs must be made. + +Boards can be distinguished by the amount of effort +spent in regulating them. First, we have the completely +open board, whose sysop is off chugging brews and +watching re-runs while his users generally degenerate +over time into peevish anarchy and eventual silence. +Second comes the supervised board, where the sysop +breaks in every once in a while to tidy up, calm brawls, +issue announcements, and rid the community of dolts +and troublemakers. Third is the heavily supervised +board, which sternly urges adult and responsible behavior +and swiftly edits any message considered offensive, +impertinent, illegal or irrelevant. And last comes +the completely edited "electronic publication," which +is presented to a silent audience which is not allowed +to respond directly in any way. + +Boards can also be grouped by their degree of anonymity. +There is the completely anonymous board, where everyone +uses pseudonyms--"handles"--and even the sysop is unaware +of the user's true identity. The sysop himself is likely +pseudonymous on a board of this type. Second, and rather +more common, is the board where the sysop knows (or thinks +he knows) the true names and addresses of all users, +but the users don't know one another's names and may not know his. +Third is the board where everyone has to use real names, +and roleplaying and pseudonymous posturing are forbidden. + +Boards can be grouped by their immediacy. "Chat-lines" +are boards linking several users together over several +different phone-lines simultaneously, so that people +exchange messages at the very moment that they type. +(Many large boards feature "chat" capabilities along +with other services.) Less immediate boards, +perhaps with a single phoneline, store messages serially, +one at a time. And some boards are only open for business +in daylight hours or on weekends, which greatly slows response. +A NETWORK of boards, such as "FidoNet," can carry electronic mail +from board to board, continent to continent, across huge distances-- +but at a relative snail's pace, so that a message can take several +days to reach its target audience and elicit a reply. + +Boards can be grouped by their degree of community. +Some boards emphasize the exchange of private, +person-to-person electronic mail. Others emphasize +public postings and may even purge people who "lurk," +merely reading posts but refusing to openly participate. +Some boards are intimate and neighborly. Others are frosty +and highly technical. Some are little more than storage +dumps for software, where users "download" and "upload" programs, +but interact among themselves little if at all. + +Boards can be grouped by their ease of access. Some boards +are entirely public. Others are private and restricted only +to personal friends of the sysop. Some boards divide users by status. +On these boards, some users, especially beginners, strangers or children, +will be restricted to general topics, and perhaps forbidden to post. +Favored users, though, are granted the ability to post as they please, +and to stay "on-line" as long as they like, even to the disadvantage +of other people trying to call in. High-status users can be given access +to hidden areas in the board, such as off-color topics, private discussions, +and/or valuable software. Favored users may even become "remote sysops" +with the power to take remote control of the board through their own +home computers. Quite often "remote sysops" end up doing all the work +and taking formal control of the enterprise, despite the fact that it's +physically located in someone else's house. Sometimes several "co-sysops" +share power. + +And boards can also be grouped by size. Massive, nationwide +commercial networks, such as CompuServe, Delphi, GEnie and Prodigy, +are run on mainframe computers and are generally not considered "boards," +though they share many of their characteristics, such as electronic mail, +discussion topics, libraries of software, and persistent and growing problems +with civil-liberties issues. Some private boards have as many as +thirty phone-lines and quite sophisticated hardware. And then +there are tiny boards. + +Boards vary in popularity. Some boards are huge and crowded, +where users must claw their way in against a constant busy-signal. +Others are huge and empty--there are few things sadder than a formerly +flourishing board where no one posts any longer, and the dead conversations +of vanished users lie about gathering digital dust. Some boards are tiny +and intimate, their telephone numbers intentionally kept confidential +so that only a small number can log on. + +And some boards are UNDERGROUND. + +Boards can be mysterious entities. The activities of +their users can be hard to differentiate from conspiracy. +Sometimes they ARE conspiracies. Boards have harbored, +or have been accused of harboring, all manner of fringe groups, +and have abetted, or been accused of abetting, every manner +of frowned-upon, sleazy, radical, and criminal activity. +There are Satanist boards. Nazi boards. Pornographic boards. +Pedophile boards. Drug- dealing boards. Anarchist boards. +Communist boards. Gay and Lesbian boards (these exist in great profusion, +many of them quite lively with well-established histories). +Religious cult boards. Evangelical boards. Witchcraft +boards, hippie boards, punk boards, skateboarder boards. +Boards for UFO believers. There may well be boards for +serial killers, airline terrorists and professional assassins. +There is simply no way to tell. Boards spring up, flourish, +and disappear in large numbers, in most every corner of +the developed world. Even apparently innocuous public +boards can, and sometimes do, harbor secret areas known +only to a few. And even on the vast, public, commercial services, +private mail is very private--and quite possibly criminal. + +Boards cover most every topic imaginable and some +that are hard to imagine. They cover a vast spectrum +of social activity. However, all board users do have +something in common: their possession of computers +and phones. Naturally, computers and phones are +primary topics of conversation on almost every board. + +And hackers and phone phreaks, those utter devotees +of computers and phones, live by boards. They swarm by boards. +They are bred by boards. By the late 1980s, phone-phreak groups +and hacker groups, united by boards, had proliferated fantastically. + + +As evidence, here is a list of hacker groups compiled +by the editors of Phrack on August 8, 1988. + + +The Administration. +Advanced Telecommunications, Inc. +ALIAS. +American Tone Travelers. +Anarchy Inc. +Apple Mafia. +The Association. +Atlantic Pirates Guild. + +Bad Ass Mother Fuckers. +Bellcore. +Bell Shock Force. +Black Bag. + +Camorra. +C&M Productions. +Catholics Anonymous. +Chaos Computer Club. +Chief Executive Officers. +Circle Of Death. +Circle Of Deneb. +Club X. +Coalition of Hi-Tech +Pirates. +Coast-To-Coast. +Corrupt Computing. +Cult Of The +Dead Cow. +Custom Retaliations. + +Damage Inc. +D&B Communications. +The Danger Gang. +Dec Hunters. +Digital Gang. +DPAK. + +Eastern Alliance. +The Elite Hackers Guild. +Elite Phreakers and Hackers Club. +The Elite Society Of America. +EPG. +Executives Of Crime. +Extasyy Elite. + +Fargo 4A. +Farmers Of Doom. +The Federation. +Feds R Us. +First Class. +Five O. +Five Star. +Force Hackers. +The 414s. + +Hack-A-Trip. +Hackers Of America. +High Mountain Hackers. +High Society. +The Hitchhikers. + +IBM Syndicate. +The Ice Pirates. +Imperial Warlords. +Inner Circle. +Inner Circle II. +Insanity Inc. +International Computer Underground Bandits. + +Justice League of America. + +Kaos Inc. +Knights Of Shadow. +Knights Of The Round Table. + +League Of Adepts. +Legion Of Doom. +Legion Of Hackers. +Lords Of Chaos. +Lunatic Labs, Unlimited. + +Master Hackers. +MAD! +The Marauders. +MD/PhD. + +Metal Communications, Inc. +MetalliBashers, Inc. +MBI. + +Metro Communications. +Midwest Pirates Guild. + +NASA Elite. +The NATO Association. +Neon Knights. + +Nihilist Order. +Order Of The Rose. +OSS. + +Pacific Pirates Guild. +Phantom Access Associates. + +PHido PHreaks. +The Phirm. +Phlash. +PhoneLine Phantoms. +Phone Phreakers Of America. +Phortune 500. + +Phreak Hack Delinquents. +Phreak Hack Destroyers. + +Phreakers, Hackers, And Laundromat Employees Gang (PHALSE Gang). +Phreaks Against Geeks. +Phreaks Against Phreaks Against Geeks. +Phreaks and Hackers of America. +Phreaks Anonymous World Wide. +Project Genesis. +The Punk Mafia. + +The Racketeers. +Red Dawn Text Files. +Roscoe Gang. + + +SABRE. +Secret Circle of Pirates. +Secret Service. +707 Club. +Shadow Brotherhood. +Sharp Inc. +65C02 Elite. + +Spectral Force. +Star League. +Stowaways. +Strata-Crackers. + + +Team Hackers '86. +Team Hackers '87. + +TeleComputist Newsletter Staff. +Tribunal Of Knowledge. + +Triple Entente. +Turn Over And Die Syndrome (TOADS). + +300 Club. +1200 Club. +2300 Club. +2600 Club. +2601 Club. + +2AF. + +The United Soft WareZ Force. +United Technical Underground. + +Ware Brigade. +The Warelords. +WASP. + +Contemplating this list is an impressive, almost humbling business. +As a cultural artifact, the thing approaches poetry. + +Underground groups--subcultures--can be distinguished +from independent cultures by their habit of referring +constantly to the parent society. Undergrounds by their +nature constantly must maintain a membrane of differentiation. +Funny/distinctive clothes and hair, specialized jargon, specialized +ghettoized areas in cities, different hours of rising, working, +sleeping. . . . The digital underground, which specializes in information, +relies very heavily on language to distinguish itself. As can be seen +from this list, they make heavy use of parody and mockery. +It's revealing to see who they choose to mock. + +First, large corporations. We have the Phortune 500, +The Chief Executive Officers, Bellcore, IBM Syndicate, +SABRE (a computerized reservation service maintained +by airlines). The common use of "Inc." is telling-- +none of these groups are actual corporations, +but take clear delight in mimicking them. + +Second, governments and police. NASA Elite, NATO Association. +"Feds R Us" and "Secret Service" are fine bits of fleering boldness. +OSS--the Office of Strategic Services was the forerunner of the CIA. + +Third, criminals. Using stigmatizing pejoratives as a perverse +badge of honor is a time-honored tactic for subcultures: +punks, gangs, delinquents, mafias, pirates, bandits, racketeers. + +Specialized orthography, especially the use of "ph" for "f" +and "z" for the plural "s," are instant recognition symbols. +So is the use of the numeral "0" for the letter "O" +--computer-software orthography generally features a +slash through the zero, making the distinction obvious. + +Some terms are poetically descriptive of computer intrusion: +the Stowaways, the Hitchhikers, the PhoneLine Phantoms, Coast-to-Coast. +Others are simple bravado and vainglorious puffery. +(Note the insistent use of the terms "elite" and "master.") +Some terms are blasphemous, some obscene, others merely cryptic-- +anything to puzzle, offend, confuse, and keep the straights at bay. + +Many hacker groups further re-encrypt their names +by the use of acronyms: United Technical Underground +becomes UTU, Farmers of Doom become FoD, the United SoftWareZ +Force becomes, at its own insistence, "TuSwF," and woe to the +ignorant rodent who capitalizes the wrong letters. + +It should be further recognized that the members of these groups +are themselves pseudonymous. If you did, in fact, run across +the "PhoneLine Phantoms," you would find them to consist of +"Carrier Culprit," "The Executioner," "Black Majik," +"Egyptian Lover," "Solid State," and "Mr Icom." +"Carrier Culprit" will likely be referred to by his friends +as "CC," as in, "I got these dialups from CC of PLP." + +It's quite possible that this entire list refers to as +few as a thousand people. It is not a complete list +of underground groups--there has never been such a list, +and there never will be. Groups rise, flourish, decline, +share membership, maintain a cloud of wannabes and +casual hangers-on. People pass in and out, are ostracized, +get bored, are busted by police, or are cornered by telco +security and presented with huge bills. Many "underground +groups" are software pirates, "warez d00dz," who might break +copy protection and pirate programs, but likely wouldn't dare +to intrude on a computer-system. + +It is hard to estimate the true population of the digital +underground. There is constant turnover. Most hackers +start young, come and go, then drop out at age 22-- +the age of college graduation. And a large majority +of "hackers" access pirate boards, adopt a handle, +swipe software and perhaps abuse a phone-code or two, +while never actually joining the elite. + +Some professional informants, who make it their business +to retail knowledge of the underground to paymasters in private +corporate security, have estimated the hacker population +at as high as fifty thousand. This is likely highly inflated, +unless one counts every single teenage software pirate +and petty phone-booth thief. My best guess is about 5,000 people. +Of these, I would guess that as few as a hundred are truly "elite" +--active computer intruders, skilled enough to penetrate +sophisticated systems and truly to worry corporate security +and law enforcement. + +Another interesting speculation is whether this group +is growing or not. Young teenage hackers are often +convinced that hackers exist in vast swarms and will soon +dominate the cybernetic universe. Older and wiser +veterans, perhaps as wizened as 24 or 25 years old, +are convinced that the glory days are long gone, that the cops +have the underground's number now, and that kids these days +are dirt-stupid and just want to play Nintendo. + +My own assessment is that computer intrusion, as a non-profit act +of intellectual exploration and mastery, is in slow decline, +at least in the United States; but that electronic fraud, +especially telecommunication crime, is growing by leaps and bounds. + +One might find a useful parallel to the digital underground +in the drug underground. There was a time, now much-obscured +by historical revisionism, when Bohemians freely shared joints +at concerts, and hip, small-scale marijuana dealers might +turn people on just for the sake of enjoying a long stoned conversation +about the Doors and Allen Ginsberg. Now drugs are increasingly verboten, +except in a high-stakes, highly-criminal world of highly addictive drugs. +Over years of disenchantment and police harassment, a vaguely ideological, +free-wheeling drug underground has relinquished the business of drug-dealing +to a far more savage criminal hard-core. This is not a pleasant prospect +to contemplate, but the analogy is fairly compelling. + +What does an underground board look like? What distinguishes +it from a standard board? It isn't necessarily the conversation-- +hackers often talk about common board topics, such as hardware, software, +sex, science fiction, current events, politics, movies, personal gossip. +Underground boards can best be distinguished by their files, or "philes," +pre-composed texts which teach the techniques and ethos of the underground. +These are prized reservoirs of forbidden knowledge. Some are anonymous, +but most proudly bear the handle of the "hacker" who has created them, +and his group affiliation, if he has one. + +Here is a partial table-of-contents of philes from an underground board, +somewhere in the heart of middle America, circa 1991. The descriptions +are mostly self-explanatory. + + +BANKAMER.ZIP 5406 06-11-91 Hacking Bank America +CHHACK.ZIP 4481 06-11-91 Chilton Hacking +CITIBANK.ZIP 4118 06-11-91 Hacking Citibank +CREDIMTC.ZIP 3241 06-11-91 Hacking Mtc Credit Company +DIGEST.ZIP 5159 06-11-91 Hackers Digest +HACK.ZIP 14031 06-11-91 How To Hack +HACKBAS.ZIP 5073 06-11-91 Basics Of Hacking +HACKDICT.ZIP 42774 06-11-91 Hackers Dictionary +HACKER.ZIP 57938 06-11-91 Hacker Info +HACKERME.ZIP 3148 06-11-91 Hackers Manual +HACKHAND.ZIP 4814 06-11-91 Hackers Handbook +HACKTHES.ZIP 48290 06-11-91 Hackers Thesis +HACKVMS.ZIP 4696 06-11-91 Hacking Vms Systems +MCDON.ZIP 3830 06-11-91 Hacking Macdonalds (Home Of The Archs) +P500UNIX.ZIP 15525 06-11-91 Phortune 500 Guide To Unix +RADHACK.ZIP 8411 06-11-91 Radio Hacking +TAOTRASH.DOC 4096 12-25-89 Suggestions For Trashing +TECHHACK.ZIP 5063 06-11-91 Technical Hacking + + +The files above are do-it-yourself manuals about computer intrusion. +The above is only a small section of a much larger library of hacking +and phreaking techniques and history. We now move into a different +and perhaps surprising area. + ++------------+ + |Anarchy| ++------------+ + +ANARC.ZIP 3641 06-11-91 Anarchy Files +ANARCHST.ZIP 63703 06-11-91 Anarchist Book +ANARCHY.ZIP 2076 06-11-91 Anarchy At Home +ANARCHY3.ZIP 6982 06-11-91 Anarchy No 3 +ANARCTOY.ZIP 2361 06-11-91 Anarchy Toys +ANTIMODM.ZIP 2877 06-11-91 Anti-modem Weapons +ATOM.ZIP 4494 06-11-91 How To Make An Atom Bomb +BARBITUA.ZIP 3982 06-11-91 Barbiturate Formula +BLCKPWDR.ZIP 2810 06-11-91 Black Powder Formulas +BOMB.ZIP 3765 06-11-91 How To Make Bombs +BOOM.ZIP 2036 06-11-91 Things That Go Boom +CHLORINE.ZIP 1926 06-11-91 Chlorine Bomb +COOKBOOK.ZIP 1500 06-11-91 Anarchy Cook Book +DESTROY.ZIP 3947 06-11-91 Destroy Stuff +DUSTBOMB.ZIP 2576 06-11-91 Dust Bomb +ELECTERR.ZIP 3230 06-11-91 Electronic Terror +EXPLOS1.ZIP 2598 06-11-91 Explosives 1 +EXPLOSIV.ZIP 18051 06-11-91 More Explosives +EZSTEAL.ZIP 4521 06-11-91 Ez-stealing +FLAME.ZIP 2240 06-11-91 Flame Thrower +FLASHLT.ZIP 2533 06-11-91 Flashlight Bomb +FMBUG.ZIP 2906 06-11-91 How To Make An Fm Bug +OMEEXPL.ZIP 2139 06-11-91 Home Explosives +HOW2BRK.ZIP 3332 06-11-91 How To Break In +LETTER.ZIP 2990 06-11-91 Letter Bomb +LOCK.ZIP 2199 06-11-91 How To Pick Locks +MRSHIN.ZIP 3991 06-11-91 Briefcase Locks +NAPALM.ZIP 3563 06-11-91 Napalm At Home +NITRO.ZIP 3158 06-11-91 Fun With Nitro +PARAMIL.ZIP 2962 06-11-91 Paramilitary Info +PICKING.ZIP 3398 06-11-91 Picking Locks +PIPEBOMB.ZIP 2137 06-11-91 Pipe Bomb +POTASS.ZIP 3987 06-11-91 Formulas With Potassium +PRANK.TXT 11074 08-03-90 More Pranks To Pull On Idiots! +REVENGE.ZIP 4447 06-11-91 Revenge Tactics +ROCKET.ZIP 2590 06-11-91 Rockets For Fun +SMUGGLE.ZIP 3385 06-11-91 How To Smuggle + +HOLY COW! The damned thing is full of stuff about bombs! + +What are we to make of this? + +First, it should be acknowledged that spreading +knowledge about demolitions to teenagers is a highly and +deliberately antisocial act. It is not, however, illegal. + +Second, it should be recognized that most of these +philes were in fact WRITTEN by teenagers. Most adult +American males who can remember their teenage years +will recognize that the notion of building a flamethrower +in your garage is an incredibly neat-o idea. ACTUALLY, +building a flamethrower in your garage, however, is +fraught with discouraging difficulty. Stuffing gunpowder +into a booby-trapped flashlight, so as to blow the arm off +your high-school vice-principal, can be a thing of dark +beauty to contemplate. Actually committing assault by +explosives will earn you the sustained attention of the +federal Bureau of Alcohol, Tobacco and Firearms. + +Some people, however, will actually try these plans. +A determinedly murderous American teenager can probably +buy or steal a handgun far more easily than he can brew +fake "napalm" in the kitchen sink. Nevertheless, +if temptation is spread before people, a certain number +will succumb, and a small minority will actually attempt +these stunts. A large minority of that small minority +will either fail or, quite likely, maim themselves, +since these "philes" have not been checked for accuracy, +are not the product of professional experience, +and are often highly fanciful. But the gloating menace +of these philes is not to be entirely dismissed. + +Hackers may not be "serious" about bombing; if they were, +we would hear far more about exploding flashlights, homemade bazookas, +and gym teachers poisoned by chlorine and potassium. +However, hackers are VERY serious about forbidden knowledge. +They are possessed not merely by curiosity, but by +a positive LUST TO KNOW. The desire to know what +others don't is scarcely new. But the INTENSITY +of this desire, as manifested by these young technophilic +denizens of the Information Age, may in fact BE new, +and may represent some basic shift in social values-- +a harbinger of what the world may come to, as society +lays more and more value on the possession, +assimilation and retailing of INFORMATION +as a basic commodity of daily life. + +There have always been young men with obsessive interests +in these topics. Never before, however, have they been able +to network so extensively and easily, and to propagandize +their interests with impunity to random passers-by. +High-school teachers will recognize that there's always +one in a crowd, but when the one in a crowd escapes control +by jumping into the phone-lines, and becomes a hundred such kids +all together on a board, then trouble is brewing visibly. +The urge of authority to DO SOMETHING, even something drastic, +is hard to resist. And in 1990, authority did something. +In fact authority did a great deal. + +# + +The process by which boards create hackers goes something +like this. A youngster becomes interested in computers-- +usually, computer games. He hears from friends that +"bulletin boards" exist where games can be obtained for free. +(Many computer games are "freeware," not copyrighted-- +invented simply for the love of it and given away to the public; +some of these games are quite good.) He bugs his parents for a modem, +or quite often, uses his parents' modem. + +The world of boards suddenly opens up. Computer games +can be quite expensive, real budget-breakers for a kid, +but pirated games, stripped of copy protection, are cheap or free. +They are also illegal, but it is very rare, almost unheard of, +for a small-scale software pirate to be prosecuted. +Once "cracked" of its copy protection, the program, +being digital data, becomes infinitely reproducible. +Even the instructions to the game, any manuals that accompany it, +can be reproduced as text files, or photocopied from legitimate sets. +Other users on boards can give many useful hints in game-playing tactics. +And a youngster with an infinite supply of free computer games can +certainly cut quite a swath among his modem-less friends. + +And boards are pseudonymous. No one need know that you're +fourteen years old--with a little practice at subterfuge, +you can talk to adults about adult things, and be accepted +and taken seriously! You can even pretend to be a girl, +or an old man, or anybody you can imagine. If you find this +kind of deception gratifying, there is ample opportunity +to hone your ability on boards. + +But local boards can grow stale. And almost every board maintains +a list of phone-numbers to other boards, some in distant, tempting, +exotic locales. Who knows what they're up to, in Oregon or Alaska +or Florida or California? It's very easy to find out--just order +the modem to call through its software--nothing to this, just typing +on a keyboard, the same thing you would do for most any computer game. +The machine reacts swiftly and in a few seconds you are talking to +a bunch of interesting people on another seaboard. + +And yet the BILLS for this trivial action can be staggering! +Just by going tippety-tap with your fingers, you may have +saddled your parents with four hundred bucks in long-distance charges, +and gotten chewed out but good. That hardly seems fair. + +How horrifying to have made friends in another state +and to be deprived of their company--and their software-- +just because telephone companies demand absurd amounts of money! +How painful, to be restricted to boards in one's own AREA CODE-- +what the heck is an "area code" anyway, and what makes it so special? +A few grumbles, complaints, and innocent questions of this sort +will often elicit a sympathetic reply from another board user-- +someone with some stolen codes to hand. You dither a while, +knowing this isn't quite right, then you make up your mind +to try them anyhow--AND THEY WORK! Suddenly you're doing something +even your parents can't do. Six months ago you were just some kid--now, +you're the Crimson Flash of Area Code 512! You're bad--you're nationwide! + +Maybe you'll stop at a few abused codes. Maybe you'll decide that +boards aren't all that interesting after all, that it's wrong, +not worth the risk --but maybe you won't. The next step +is to pick up your own repeat-dialling program-- +to learn to generate your own stolen codes. +(This was dead easy five years ago, much harder +to get away with nowadays, but not yet impossible.) +And these dialling programs are not complex or intimidating-- +some are as small as twenty lines of software. + +Now, you too can share codes. You can trade codes to learn +other techniques. If you're smart enough to catch on, +and obsessive enough to want to bother, and ruthless enough +to start seriously bending rules, then you'll get better, fast. +You start to develop a rep. You move up to a heavier class +of board--a board with a bad attitude, the kind of board +that naive dopes like your classmates and your former self +have never even heard of! You pick up the jargon of phreaking +and hacking from the board. You read a few of those anarchy philes-- +and man, you never realized you could be a real OUTLAW without +ever leaving your bedroom. + +You still play other computer games, but now you have a new +and bigger game. This one will bring you a different kind of status +than destroying even eight zillion lousy space invaders. + +Hacking is perceived by hackers as a "game." This is +not an entirely unreasonable or sociopathic perception. +You can win or lose at hacking, succeed or fail, +but it never feels "real." It's not simply that +imaginative youngsters sometimes have a hard time +telling "make-believe" from "real life." Cyberspace +is NOT REAL! "Real" things are physical objects +like trees and shoes and cars. Hacking takes place +on a screen. Words aren't physical, numbers +(even telephone numbers and credit card numbers) +aren't physical. Sticks and stones may break my bones, +but data will never hurt me. Computers SIMULATE reality, +like computer games that simulate tank battles or dogfights +or spaceships. Simulations are just make-believe, +and the stuff in computers is NOT REAL. + +Consider this: if "hacking" is supposed to be so serious and +real-life and dangerous, then how come NINE-YEAR-OLD KIDS have +computers and modems? You wouldn't give a nine year old his own car, +or his own rifle, or his own chainsaw--those things are "real." + +People underground are perfectly aware that the "game" +is frowned upon by the powers that be. Word gets around +about busts in the underground. Publicizing busts is one +of the primary functions of pirate boards, but they also +promulgate an attitude about them, and their own idiosyncratic +ideas of justice. The users of underground boards won't complain +if some guy is busted for crashing systems, spreading viruses, +or stealing money by wire-fraud. They may shake their heads +with a sneaky grin, but they won't openly defend these practices. +But when a kid is charged with some theoretical amount of theft: +$233,846.14, for instance, because he sneaked into a computer +and copied something, and kept it in his house on a floppy disk-- +this is regarded as a sign of near-insanity from prosecutors, +a sign that they've drastically mistaken the immaterial game +of computing for their real and boring everyday world +of fatcat corporate money. + +It's as if big companies and their suck-up lawyers +think that computing belongs to them, and they can +retail it with price stickers, as if it were boxes +of laundry soap! But pricing "information" is like +trying to price air or price dreams. Well, anybody +on a pirate board knows that computing can be, +and ought to be, FREE. Pirate boards are little +independent worlds in cyberspace, and they don't belong +to anybody but the underground. Underground boards +aren't "brought to you by Procter & Gamble." + +To log on to an underground board can mean to +experience liberation, to enter a world where, +for once, money isn't everything and adults +don't have all the answers. + +Let's sample another vivid hacker manifesto. Here are +some excerpts from "The Conscience of a Hacker," by "The Mentor," +from Phrack Volume One, Issue 7, Phile 3. + +"I made a discovery today. I found a computer. +Wait a second, this is cool. It does what I want it to. +If it makes a mistake, it's because I screwed it up. +Not because it doesn't like me. (. . .) +"And then it happened. . .a door opened to a world. . . +rushing through the phone line like heroin through an +addict's veins, an electronic pulse is sent out, +a refuge from day-to-day incompetencies is sought. . . +a board is found. `This is it. . .this is where I belong. . .' +"I know everyone here. . .even if I've never met them, +never talked to them, may never hear from them again. . . +I know you all. . . (. . .) + +"This is our world now. . .the world of the electron +and the switch, the beauty of the baud. We make use of a +service already existing without paying for what could be +dirt-cheap if it wasn't run by profiteering gluttons, and you +call us criminals. We explore. . .and you call us criminals. +We seek after knowledge. . .and you call us criminals. +We exist without skin color, without nationality, +without religious bias. . .and you call us criminals. +You build atomic bombs, you wage wars, you murder, +cheat and lie to us and try to make us believe that +it's for our own good, yet we're the criminals. + +"Yes, I am a criminal. My crime is that of curiosity. +My crime is that of judging people by what they say and think, +not what they look like. My crime is that of outsmarting you, +something that you will never forgive me for." + +# + +There have been underground boards almost as long +as there have been boards. One of the first was 8BBS, +which became a stronghold of the West Coast phone-phreak elite. +After going on-line in March 1980, 8BBS sponsored "Susan Thunder," +and "Tuc," and, most notoriously, "the Condor." "The Condor" +bore the singular distinction of becoming the most vilified +American phreak and hacker ever. Angry underground associates, +fed up with Condor's peevish behavior, turned him in to police, +along with a heaping double-helping of outrageous hacker legendry. +As a result, Condor was kept in solitary confinement for seven months, +for fear that he might start World War Three by triggering missile silos +from the prison payphone. (Having served his time, Condor is now +walking around loose; WWIII has thus far conspicuously failed to occur.) + +The sysop of 8BBS was an ardent free-speech enthusiast +who simply felt that ANY attempt to restrict the expression +of his users was unconstitutional and immoral. +Swarms of the technically curious entered 8BBS +and emerged as phreaks and hackers, until, in 1982, +a friendly 8BBS alumnus passed the sysop a new modem +which had been purchased by credit-card fraud. +Police took this opportunity to seize the entire board +and remove what they considered an attractive nuisance. + +Plovernet was a powerful East Coast pirate board +that operated in both New York and Florida. +Owned and operated by teenage hacker "Quasi Moto," +Plovernet attracted five hundred eager users in 1983. +"Emmanuel Goldstein" was one-time co-sysop of Plovernet, +along with "Lex Luthor," founder of the "Legion of Doom" group. +Plovernet bore the signal honor of being the original home +of the "Legion of Doom," about which the reader will be hearing +a great deal, soon. + +"Pirate-80," or "P-80," run by a sysop known as "Scan-Man," +got into the game very early in Charleston, and continued +steadily for years. P-80 flourished so flagrantly that +even its most hardened users became nervous, and some +slanderously speculated that "Scan Man" must have ties +to corporate security, a charge he vigorously denied. + +"414 Private" was the home board for the first GROUP +to attract conspicuous trouble, the teenage "414 Gang," +whose intrusions into Sloan-Kettering Cancer Center and +Los Alamos military computers were to be a nine-days-wonder in 1982. + +At about this time, the first software piracy boards +began to open up, trading cracked games for the Atari 800 +and the Commodore C64. Naturally these boards were +heavily frequented by teenagers. And with the 1983 +release of the hacker-thriller movie War Games, +the scene exploded. It seemed that every kid +in America had demanded and gotten a modem for Christmas. +Most of these dabbler wannabes put their modems in the attic +after a few weeks, and most of the remainder minded their +P's and Q's and stayed well out of hot water. But some +stubborn and talented diehards had this hacker kid in +War Games figured for a happening dude. They simply +could not rest until they had contacted the underground-- +or, failing that, created their own. + +In the mid-80s, underground boards sprang up like digital fungi. +ShadowSpawn Elite. Sherwood Forest I, II, and III. +Digital Logic Data Service in Florida, sysoped by no less +a man than "Digital Logic" himself; Lex Luthor of the +Legion of Doom was prominent on this board, since it +was in his area code. Lex's own board, "Legion of Doom," +started in 1984. The Neon Knights ran a network of Apple- +hacker boards: Neon Knights North, South, East and West. +Free World II was run by "Major Havoc." Lunatic Labs +is still in operation as of this writing. Dr. Ripco +in Chicago, an anything-goes anarchist board with an +extensive and raucous history, was seized by Secret Service +agents in 1990 on Sundevil day, but up again almost immediately, +with new machines and scarcely diminished vigor. + +The St. Louis scene was not to rank with major centers +of American hacking such as New York and L.A. But St. +Louis did rejoice in possession of "Knight Lightning" +and "Taran King," two of the foremost JOURNALISTS native +to the underground. Missouri boards like Metal Shop, +Metal Shop Private, Metal Shop Brewery, may not have +been the heaviest boards around in terms of illicit +expertise. But they became boards where hackers could +exchange social gossip and try to figure out what the +heck was going on nationally--and internationally. +Gossip from Metal Shop was put into the form of news files, +then assembled into a general electronic publication, +Phrack, a portmanteau title coined from "phreak" and "hack." +The Phrack editors were as obsessively curious about other +hackers as hackers were about machines. + +Phrack, being free of charge and lively reading, began +to circulate throughout the underground. As Taran King +and Knight Lightning left high school for college, +Phrack began to appear on mainframe machines linked to BITNET, +and, through BITNET to the "Internet," that loose but +extremely potent not-for-profit network where academic, +governmental and corporate machines trade data through +the UNIX TCP/IP protocol. (The "Internet Worm" of +November 2-3,1988, created by Cornell grad student Robert Morris, +was to be the largest and best-publicized computer-intrusion scandal +to date. Morris claimed that his ingenious "worm" program was meant +to harmlessly explore the Internet, but due to bad programming, +the Worm replicated out of control and crashed some six thousand +Internet computers. Smaller-scale and less ambitious Internet hacking +was a standard for the underground elite.) + +Most any underground board not hopelessly lame and out-of-it +would feature a complete run of Phrack--and, possibly, +the lesser-known standards of the underground: +the Legion of Doom Technical Journal, the obscene +and raucous Cult of the Dead Cow files, P/HUN magazine, +Pirate, the Syndicate Reports, and perhaps the highly +anarcho-political Activist Times Incorporated. + +Possession of Phrack on one's board was prima facie +evidence of a bad attitude. Phrack was seemingly everywhere, +aiding, abetting, and spreading the underground ethos. +And this did not escape the attention of corporate security +or the police. + +We now come to the touchy subject of police and boards. +Police, do, in fact, own boards. In 1989, there were +police-sponsored boards in California, Colorado, Florida, +Georgia, Idaho, Michigan, Missouri, Texas, and Virginia: +boards such as "Crime Bytes," "Crimestoppers," "All Points" +and "Bullet-N-Board." Police officers, as private computer +enthusiasts, ran their own boards in Arizona, California, +Colorado, Connecticut, Florida, Missouri, Maryland, +New Mexico, North Carolina, Ohio, Tennessee and Texas. +Police boards have often proved helpful in community relations. +Sometimes crimes are reported on police boards. + +Sometimes crimes are COMMITTED on police boards. +This has sometimes happened by accident, as naive hackers +blunder onto police boards and blithely begin offering telephone codes. +Far more often, however, it occurs through the now almost-traditional +use of "sting boards." The first police sting-boards were established +in 1985: "Underground Tunnel" in Austin, Texas, whose sysop +Sgt. Robert Ansley called himself "Pluto"--"The Phone Company" +in Phoenix, Arizona, run by Ken MacLeod of the Maricopa County +Sheriff's office--and Sgt. Dan Pasquale's board in Fremont, California. +Sysops posed as hackers, and swiftly garnered coteries of ardent users, +who posted codes and loaded pirate software with abandon, +and came to a sticky end. + +Sting boards, like other boards, are cheap to operate, +very cheap by the standards of undercover police operations. +Once accepted by the local underground, sysops will likely be +invited into other pirate boards, where they can compile more dossiers. +And when the sting is announced and the worst offenders arrested, +the publicity is generally gratifying. The resultant paranoia +in the underground--perhaps more justly described as a "deterrence effect"-- +tends to quell local lawbreaking for quite a while. + +Obviously police do not have to beat the underbrush for hackers. +On the contrary, they can go trolling for them. Those caught +can be grilled. Some become useful informants. They can lead +the way to pirate boards all across the country. + +And boards all across the country showed the sticky +fingerprints of Phrack, and of that loudest and most +flagrant of all underground groups, the "Legion of Doom." + +The term "Legion of Doom" came from comic books. The Legion of Doom, +a conspiracy of costumed super- villains headed by the chrome-domed +criminal ultra- mastermind Lex Luthor, gave Superman a lot of four-color +graphic trouble for a number of decades. Of course, Superman, +that exemplar of Truth, Justice, and the American Way, +always won in the long run. This didn't matter to the hacker Doomsters-- +"Legion of Doom" was not some thunderous and evil Satanic reference, +it was not meant to be taken seriously. "Legion of Doom" came +from funny-books and was supposed to be funny. + +"Legion of Doom" did have a good mouthfilling ring to it, though. +It sounded really cool. Other groups, such as the "Farmers of Doom," +closely allied to LoD, recognized this grandiloquent quality, +and made fun of it. There was even a hacker group called +"Justice League of America," named after Superman's club +of true-blue crimefighting superheros. + +But they didn't last; the Legion did. + +The original Legion of Doom, hanging out on Quasi Moto's Plovernet board, +were phone phreaks. They weren't much into computers. "Lex Luthor" himself +(who was under eighteen when he formed the Legion) was a COSMOS expert, +COSMOS being the "Central System for Mainframe Operations," +a telco internal computer network. Lex would eventually become +quite a dab hand at breaking into IBM mainframes, but although +everyone liked Lex and admired his attitude, he was not considered +a truly accomplished computer intruder. Nor was he the "mastermind" +of the Legion of Doom--LoD were never big on formal leadership. +As a regular on Plovernet and sysop of his "Legion of Doom BBS," +Lex was the Legion's cheerleader and recruiting officer. + +Legion of Doom began on the ruins of an earlier phreak group, +The Knights of Shadow. Later, LoD was to subsume the personnel +of the hacker group "Tribunal of Knowledge." People came and went +constantly in LoD; groups split up or formed offshoots. + +Early on, the LoD phreaks befriended a few computer-intrusion +enthusiasts, who became the associated "Legion of Hackers." +Then the two groups conflated into the "Legion of Doom/Hackers," +or LoD/H. When the original "hacker" wing, Messrs. "Compu-Phreak" +and "Phucked Agent 04," found other matters to occupy their time, +the extra "/H" slowly atrophied out of the name; but by this time +the phreak wing, Messrs. Lex Luthor, "Blue Archer," "Gary Seven," +"Kerrang Khan," "Master of Impact," "Silver Spy," "The Marauder," +and "The Videosmith," had picked up a plethora of intrusion +expertise and had become a force to be reckoned with. + +LoD members seemed to have an instinctive understanding +that the way to real power in the underground lay through +covert publicity. LoD were flagrant. Not only was it one +of the earliest groups, but the members took pains to widely +distribute their illicit knowledge. Some LoD members, +like "The Mentor," were close to evangelical about it. +Legion of Doom Technical Journal began to show up on boards +throughout the underground. + +LoD Technical Journal was named in cruel parody +of the ancient and honored AT&T Technical Journal. +The material in these two publications was quite similar-- +much of it, adopted from public journals and discussions +in the telco community. And yet, the predatory attitude +of LoD made even its most innocuous data seem deeply sinister; +an outrage; a clear and present danger. + +To see why this should be, let's consider the following +(invented) paragraphs, as a kind of thought experiment. + +(A) "W. Fred Brown, AT&T Vice President for +Advanced Technical Development, testified May 8 +at a Washington hearing of the National Telecommunications +and Information Administration (NTIA), regarding +Bellcore's GARDEN project. GARDEN (Generalized +Automatic Remote Distributed Electronic Network) is a +telephone-switch programming tool that makes it possible +to develop new telecom services, including hold-on-hold +and customized message transfers, from any keypad terminal, +within seconds. The GARDEN prototype combines centrex +lines with a minicomputer using UNIX operating system software." + +(B) "Crimson Flash 512 of the Centrex Mobsters reports: +D00dz, you wouldn't believe this GARDEN bullshit Bellcore's +just come up with! Now you don't even need a lousy Commodore +to reprogram a switch--just log on to GARDEN as a technician, +and you can reprogram switches right off the keypad in any +public phone booth! You can give yourself hold-on-hold +and customized message transfers, and best of all, +the thing is run off (notoriously insecure) centrex lines +using--get this--standard UNIX software! Ha ha ha ha!" + +Message (A), couched in typical techno-bureaucratese, +appears tedious and almost unreadable. (A) scarcely seems +threatening or menacing. Message (B), on the other hand, +is a dreadful thing, prima facie evidence of a dire conspiracy, +definitely not the kind of thing you want your teenager reading. + +The INFORMATION, however, is identical. It is PUBLIC +information, presented before the federal government in +an open hearing. It is not "secret." It is not "proprietary." +It is not even "confidential." On the contrary, the +development of advanced software systems is a matter +of great public pride to Bellcore. + +However, when Bellcore publicly announces a project of this kind, +it expects a certain attitude from the public--something along +the lines of GOSH WOW, YOU GUYS ARE GREAT, KEEP THAT UP, WHATEVER IT IS-- +certainly not cruel mimickry, one-upmanship and outrageous speculations +about possible security holes. + +Now put yourself in the place of a policeman confronted by +an outraged parent, or telco official, with a copy of Version (B). +This well-meaning citizen, to his horror, has discovered +a local bulletin-board carrying outrageous stuff like (B), +which his son is examining with a deep and unhealthy interest. +If (B) were printed in a book or magazine, you, as an American +law enforcement officer, would know that it would take +a hell of a lot of trouble to do anything about it; +but it doesn't take technical genius to recognize that +if there's a computer in your area harboring stuff like (B), +there's going to be trouble. + +In fact, if you ask around, any computer-literate cop +will tell you straight out that boards with stuff like (B) +are the SOURCE of trouble. And the WORST source of trouble +on boards are the ringleaders inventing and spreading stuff like (B). +If it weren't for these jokers, there wouldn't BE any trouble. + +And Legion of Doom were on boards like nobody else. +Plovernet. The Legion of Doom Board. The Farmers of Doom Board. +Metal Shop. OSUNY. Blottoland. Private Sector. Atlantis. +Digital Logic. Hell Phrozen Over. + +LoD members also ran their own boards. "Silver Spy" started +his own board, "Catch-22," considered one of the heaviest around. +So did "Mentor," with his "Phoenix Project." When they didn't run boards +themselves, they showed up on other people's boards, to brag, boast, +and strut. And where they themselves didn't go, their philes went, +carrying evil knowledge and an even more evil attitude. + +As early as 1986, the police were under the vague impression +that EVERYONE in the underground was Legion of Doom. +LoD was never that large--considerably smaller than either +"Metal Communications" or "The Administration," for instance-- +but LoD got tremendous press. Especially in Phrack, +which at times read like an LoD fan magazine; and Phrack +was everywhere, especially in the offices of telco security. +You couldn't GET busted as a phone phreak, a hacker, +or even a lousy codes kid or warez dood, without the cops +asking if you were LoD. + +This was a difficult charge to deny, as LoD never +distributed membership badges or laminated ID cards. +If they had, they would likely have died out quickly, +for turnover in their membership was considerable. +LoD was less a high-tech street-gang than an ongoing +state-of-mind. LoD was the Gang That Refused to Die. +By 1990, LoD had RULED for ten years, and it seemed WEIRD +to police that they were continually busting people who were +only sixteen years old. All these teenage small-timers +were pleading the tiresome hacker litany of "just curious, +no criminal intent." Somewhere at the center of this +conspiracy there had to be some serious adult masterminds, +not this seemingly endless supply of myopic suburban +white kids with high SATs and funny haircuts. + +There was no question that most any American hacker +arrested would "know" LoD. They knew the handles +of contributors to LoD Tech Journal, and were likely +to have learned their craft through LoD boards and LoD activism. +But they'd never met anyone from LoD. Even some of the +rotating cadre who were actually and formally "in LoD" +knew one another only by board-mail and pseudonyms. +This was a highly unconventional profile for a criminal conspiracy. +Computer networking, and the rapid evolution of the digital underground, +made the situation very diffuse and confusing. + +Furthermore, a big reputation in the digital underground +did not coincide with one's willingness to commit "crimes." +Instead, reputation was based on cleverness and technical mastery. +As a result, it often seemed that the HEAVIER the hackers were, +the LESS likely they were to have committed any kind of common, +easily prosecutable crime. There were some hackers who could really steal. +And there were hackers who could really hack. But the two groups didn't seem +to overlap much, if at all. For instance, most people in the underground +looked up to "Emmanuel Goldstein" of 2600 as a hacker demigod. +But Goldstein's publishing activities were entirely legal-- +Goldstein just printed dodgy stuff and talked about politics, +he didn't even hack. When you came right down to it, +Goldstein spent half his time complaining that computer security +WASN'T STRONG ENOUGH and ought to be drastically improved +across the board! + +Truly heavy-duty hackers, those with serious technical skills +who had earned the respect of the underground, never stole money +or abused credit cards. Sometimes they might abuse phone-codes-- +but often, they seemed to get all the free phone-time they wanted +without leaving a trace of any kind. + +The best hackers, the most powerful and technically accomplished, +were not professional fraudsters. They raided computers habitually, +but wouldn't alter anything, or damage anything. They didn't even steal +computer equipment--most had day-jobs messing with hardware, +and could get all the cheap secondhand equipment they wanted. +The hottest hackers, unlike the teenage wannabes, weren't snobs +about fancy or expensive hardware. Their machines tended to be +raw second-hand digital hot-rods full of custom add-ons that +they'd cobbled together out of chickenwire, memory chips and spit. +Some were adults, computer software writers and consultants by trade, +and making quite good livings at it. Some of them ACTUALLY WORKED +FOR THE PHONE COMPANY--and for those, the "hackers" actually found +under the skirts of Ma Bell, there would be little mercy in 1990. + +It has long been an article of faith in the +underground that the "best" hackers never get caught. +They're far too smart, supposedly. They never get caught +because they never boast, brag, or strut. These demigods +may read underground boards (with a condescending smile), +but they never say anything there. The "best" hackers, +according to legend, are adult computer professionals, +such as mainframe system administrators, who already know +the ins and outs of their particular brand of security. +Even the "best" hacker can't break in to just any computer at random: +the knowledge of security holes is too specialized, varying widely +with different software and hardware. But if people are employed to run, +say, a UNIX mainframe or a VAX/VMS machine, then they tend to learn +security from the inside out. Armed with this knowledge, +they can look into most anybody else's UNIX or VMS +without much trouble or risk, if they want to. +And, according to hacker legend, of course they want to, +so of course they do. They just don't make a big deal +of what they've done. So nobody ever finds out. + +It is also an article of faith in the underground that +professional telco people "phreak" like crazed weasels. +OF COURSE they spy on Madonna's phone calls--I mean, +WOULDN'T YOU? Of course they give themselves free long- +distance--why the hell should THEY pay, they're running +the whole shebang! + +It has, as a third matter, long been an article of faith +that any hacker caught can escape serious punishment if +he confesses HOW HE DID IT. Hackers seem to believe +that governmental agencies and large corporations are +blundering about in cyberspace like eyeless jellyfish +or cave salamanders. They feel that these large +but pathetically stupid organizations will proffer up +genuine gratitude, and perhaps even a security post +and a big salary, to the hot-shot intruder who will deign +to reveal to them the supreme genius of his modus operandi. + +In the case of longtime LoD member "Control-C," +this actually happened, more or less. Control-C had led +Michigan Bell a merry chase, and when captured in 1987, +he turned out to be a bright and apparently physically +harmless young fanatic, fascinated by phones. There was +no chance in hell that Control-C would actually repay the +enormous and largely theoretical sums in long-distance +service that he had accumulated from Michigan Bell. +He could always be indicted for fraud or computer-intrusion, +but there seemed little real point in this--he hadn't +physically damaged any computer. He'd just plead guilty, +and he'd likely get the usual slap-on-the-wrist, +and in the meantime it would be a big hassle for Michigan Bell +just to bring up the case. But if kept on the payroll, +he might at least keep his fellow hackers at bay. + +There were uses for him. For instance, a contrite +Control-C was featured on Michigan Bell internal posters, +sternly warning employees to shred their trash. +He'd always gotten most of his best inside info from +"trashing"--raiding telco dumpsters, for useful data +indiscreetly thrown away. He signed these posters, too. +Control-C had become something like a Michigan Bell mascot. +And in fact, Control-C DID keep other hackers at bay. +Little hackers were quite scared of Control-C and his +heavy-duty Legion of Doom friends. And big hackers WERE +his friends and didn't want to screw up his cushy situation. + +No matter what one might say of LoD, they did stick together. +When "Wasp," an apparently genuinely malicious New York hacker, +began crashing Bellcore machines, Control-C received swift volunteer +help from "the Mentor" and the Georgia LoD wing made up of +"The Prophet," "Urvile," and "Leftist." Using Mentor's Phoenix +Project board to coordinate, the Doomsters helped telco security +to trap Wasp, by luring him into a machine with a tap +and line-trace installed. Wasp lost. LoD won! And my, did they brag. + +Urvile, Prophet and Leftist were well-qualified for this activity, +probably more so even than the quite accomplished Control-C. +The Georgia boys knew all about phone switching-stations. +Though relative johnny-come-latelies in the Legion of Doom, +they were considered some of LoD's heaviest guys, +into the hairiest systems around. They had the good fortune +to live in or near Atlanta, home of the sleepy and apparently +tolerant BellSouth RBOC. + +As RBOC security went, BellSouth were "cake." US West (of Arizona, +the Rockies and the Pacific Northwest) were tough and aggressive, +probably the heaviest RBOC around. Pacific Bell, California's PacBell, +were sleek, high-tech, and longtime veterans of the LA phone-phreak wars. +NYNEX had the misfortune to run the New York City area, and were warily +prepared for most anything. Even Michigan Bell, a division of the +Ameritech RBOC, at least had the elementary sense to hire their own hacker +as a useful scarecrow. But BellSouth, even though their corporate P.R. +proclaimed them to have "Everything You Expect From a Leader," were pathetic. + +When rumor about LoD's mastery of Georgia's switching network got around +to BellSouth through Bellcore and telco security scuttlebutt, +they at first refused to believe it. If you paid serious attention +to every rumor out and about these hacker kids, you would hear all kinds +of wacko saucer-nut nonsense: that the National Security Agency +monitored all American phone calls, that the CIA and DEA tracked +traffic on bulletin-boards with word-analysis programs, +that the Condor could start World War III from a payphone. + +If there were hackers into BellSouth switching-stations, then how come +nothing had happened? Nothing had been hurt. BellSouth's machines +weren't crashing. BellSouth wasn't suffering especially badly from fraud. +BellSouth's customers weren't complaining. BellSouth was headquartered +in Atlanta, ambitious metropolis of the new high-tech Sunbelt; +and BellSouth was upgrading its network by leaps and bounds, +digitizing the works left right and center. They could hardly be +considered sluggish or naive. BellSouth's technical expertise +was second to none, thank you kindly. But then came the Florida business. + +On June 13, 1989, callers to the Palm Beach County Probation Department, +in Delray Beach, Florida, found themselves involved in a remarkable +discussion with a phone-sex worker named "Tina" in New York State. +Somehow, ANY call to this probation office near Miami was instantly +and magically transported across state lines, at no extra charge to the user, +to a pornographic phone-sex hotline hundreds of miles away! + +This practical joke may seem utterly hilarious at first hearing, +and indeed there was a good deal of chuckling about it in +phone phreak circles, including the Autumn 1989 issue of 2600. +But for Southern Bell (the division of the BellSouth RBOC +supplying local service for Florida, Georgia, North Carolina +and South Carolina), this was a smoking gun. For the first time ever, +a computer intruder had broken into a BellSouth central office +switching station and re-programmed it! + +Or so BellSouth thought in June 1989. Actually, LoD members had been +frolicking harmlessly in BellSouth switches since September 1987. +The stunt of June 13--call-forwarding a number through manipulation +of a switching station--was child's play for hackers as accomplished +as the Georgia wing of LoD. Switching calls interstate sounded like +a big deal, but it took only four lines of code to accomplish this. +An easy, yet more discreet, stunt, would be to call-forward another +number to your own house. If you were careful and considerate, +and changed the software back later, then not a soul would know. +Except you. And whoever you had bragged to about it. + +As for BellSouth, what they didn't know wouldn't hurt them. + +Except now somebody had blown the whole thing wide open, and BellSouth knew. + +A now alerted and considerably paranoid BellSouth began searching switches +right and left for signs of impropriety, in that hot summer of 1989. +No fewer than forty-two BellSouth employees were put on 12-hour shifts, +twenty-four hours a day, for two solid months, poring over records +and monitoring computers for any sign of phony access. These forty-two +overworked experts were known as BellSouth's "Intrusion Task Force." + +What the investigators found astounded them. Proprietary telco databases +had been manipulated: phone numbers had been created out of thin air, +with no users' names and no addresses. And perhaps worst of all, +no charges and no records of use. The new digital ReMOB (Remote Observation) +diagnostic feature had been extensively tampered with--hackers had learned to +reprogram ReMOB software, so that they could listen in on any switch-routed +call at their leisure! They were using telco property to SPY! + +The electrifying news went out throughout law enforcement in 1989. +It had never really occurred to anyone at BellSouth that their prized +and brand-new digital switching-stations could be RE-PROGRAMMED. +People seemed utterly amazed that anyone could have the nerve. +Of course these switching stations were "computers," and everybody +knew hackers liked to "break into computers:" but telephone people's +computers were DIFFERENT from normal people's computers. + +The exact reason WHY these computers were "different" was +rather ill-defined. It certainly wasn't the extent of their security. +The security on these BellSouth computers was lousy; the AIMSX computers, +for instance, didn't even have passwords. But there was no question that +BellSouth strongly FELT that their computers were very different indeed. +And if there were some criminals out there who had not gotten that message, +BellSouth was determined to see that message taught. + +After all, a 5ESS switching station was no mere bookkeeping system for +some local chain of florists. Public service depended on these stations. +Public SAFETY depended on these stations. + +And hackers, lurking in there call-forwarding or ReMobbing, could spy +on anybody in the local area! They could spy on telco officials! +They could spy on police stations! They could spy on local offices +of the Secret Service. . . . + +In 1989, electronic cops and hacker-trackers began using scrambler-phones +and secured lines. It only made sense. There was no telling who was into +those systems. Whoever they were, they sounded scary. This was some +new level of antisocial daring. Could be West German hackers, in the pay +of the KGB. That too had seemed a weird and farfetched notion, +until Clifford Stoll had poked and prodded a sluggish Washington +law-enforcement bureaucracy into investigating a computer intrusion +that turned out to be exactly that--HACKERS, IN THE PAY OF THE KGB! +Stoll, the systems manager for an Internet lab in Berkeley California, +had ended up on the front page of the New Nork Times, proclaimed a national +hero in the first true story of international computer espionage. +Stoll's counterspy efforts, which he related in a bestselling book, +The Cuckoo's Egg, in 1989, had established the credibility of `hacking' +as a possible threat to national security. The United States Secret Service +doesn't mess around when it suspects a possible action by a foreign +intelligence apparat. + +The Secret Service scrambler-phones and secured lines put +a tremendous kink in law enforcement's ability to operate freely; +to get the word out, cooperate, prevent misunderstandings. +Nevertheless, 1989 scarcely seemed the time for half-measures. +If the police and Secret Service themselves were not operationally secure, +then how could they reasonably demand measures of security from +private enterprise? At least, the inconvenience made people aware +of the seriousness of the threat. + +If there was a final spur needed to get the police off the dime, +it came in the realization that the emergency 911 system was vulnerable. +The 911 system has its own specialized software, but it is run on the same +digital switching systems as the rest of the telephone network. +911 is not physically different from normal telephony. But it is +certainly culturally different, because this is the area of +telephonic cyberspace reserved for the police and emergency services. + +Your average policeman may not know much about hackers or phone-phreaks. +Computer people are weird; even computer COPS are rather weird; +the stuff they do is hard to figure out. But a threat to the 911 system +is anything but an abstract threat. If the 911 system goes, people can die. + +Imagine being in a car-wreck, staggering to a phone-booth, +punching 911 and hearing "Tina" pick up the phone-sex line +somewhere in New York! The situation's no longer comical, somehow. + +And was it possible? No question. Hackers had attacked 911 +systems before. Phreaks can max-out 911 systems just by siccing +a bunch of computer-modems on them in tandem, dialling them over +and over until they clog. That's very crude and low-tech, +but it's still a serious business. + +The time had come for action. It was time to take stern measures +with the underground. It was time to start picking up the dropped threads, +the loose edges, the bits of braggadocio here and there; it was time to get +on the stick and start putting serious casework together. Hackers weren't +"invisible." They THOUGHT they were invisible; but the truth was, +they had just been tolerated too long. + +Under sustained police attention in the summer of '89, the digital +underground began to unravel as never before. + +The first big break in the case came very early on: July 1989, +the following month. The perpetrator of the "Tina" switch was caught, +and confessed. His name was "Fry Guy," a 16-year-old in Indiana. +Fry Guy had been a very wicked young man. + +Fry Guy had earned his handle from a stunt involving French fries. +Fry Guy had filched the log-in of a local MacDonald's manager +and had logged-on to the MacDonald's mainframe on the Sprint +Telenet system. Posing as the manager, Fry Guy had altered +MacDonald's records, and given some teenage hamburger-flipping +friends of his, generous raises. He had not been caught. + +Emboldened by success, Fry Guy moved on to credit-card abuse. +Fry Guy was quite an accomplished talker; with a gift for +"social engineering." If you can do "social engineering" +--fast-talk, fake-outs, impersonation, conning, scamming-- +then card abuse comes easy. (Getting away with it in +the long run is another question). + +Fry Guy had run across "Urvile" of the Legion of Doom +on the ALTOS Chat board in Bonn, Germany. ALTOS Chat +was a sophisticated board, accessible through globe-spanning +computer networks like BITnet, Tymnet, and Telenet. +ALTOS was much frequented by members of Germany's +Chaos Computer Club. Two Chaos hackers who hung out on ALTOS, +"Jaeger" and "Pengo," had been the central villains of +Clifford Stoll's Cuckoo's Egg case: consorting in East Berlin +with a spymaster from the KGB, and breaking into American +computers for hire, through the Internet. + +When LoD members learned the story of Jaeger's depredations +from Stoll's book, they were rather less than impressed, +technically speaking. On LoD's own favorite board of the moment, +"Black Ice," LoD members bragged that they themselves could have done +all the Chaos break-ins in a week flat! Nevertheless, LoD were grudgingly +impressed by the Chaos rep, the sheer hairy-eyed daring of hash-smoking +anarchist hackers who had rubbed shoulders with the fearsome big-boys +of international Communist espionage. LoD members sometimes traded +bits of knowledge with friendly German hackers on ALTOS--phone numbers +for vulnerable VAX/VMS computers in Georgia, for instance. +Dutch and British phone phreaks, and the Australian clique of +"Phoenix," "Nom," and "Electron," were ALTOS regulars, too. +In underground circles, to hang out on ALTOS was considered +the sign of an elite dude, a sophisticated hacker of the +international digital jet-set. + +Fry Guy quickly learned how to raid information from credit-card +consumer-reporting agencies. He had over a hundred stolen credit-card +numbers in his notebooks, and upwards of a thousand swiped long-distance +access codes. He knew how to get onto Altos, and how to talk the talk of +the underground convincingly. He now wheedled knowledge of switching-station +tricks from Urvile on the ALTOS system. + +Combining these two forms of knowledge enabled Fry Guy to bootstrap +his way up to a new form of wire-fraud. First, he'd snitched credit card +numbers from credit-company computers. The data he copied included names, +addresses and phone numbers of the random card-holders. + +Then Fry Guy, impersonating a card-holder, called up Western Union +and asked for a cash advance on "his" credit card. Western Union, +as a security guarantee, would call the customer back, at home, +to verify the transaction. + +But, just as he had switched the Florida probation office to "Tina" +in New York, Fry Guy switched the card-holder's number to a local pay-phone. +There he would lurk in wait, muddying his trail by routing and re-routing +the call, through switches as far away as Canada. When the call came through, +he would boldly "social-engineer," or con, the Western Union people, pretending +to be the legitimate card-holder. Since he'd answered the proper phone number, +the deception was not very hard. Western Union's money was then shipped to +a confederate of Fry Guy's in his home town in Indiana. + +Fry Guy and his cohort, using LoD techniques, stole six thousand dollars +from Western Union between December 1988 and July 1989. They also dabbled +in ordering delivery of stolen goods through card-fraud. Fry Guy +was intoxicated with success. The sixteen-year-old fantasized wildly +to hacker rivals, boasting that he'd used rip-off money to hire himself +a big limousine, and had driven out-of-state with a groupie from +his favorite heavy-metal band, Motley Crue. + +Armed with knowledge, power, and a gratifying stream of free money, +Fry Guy now took it upon himself to call local representatives +of Indiana Bell security, to brag, boast, strut, and utter +tormenting warnings that his powerful friends in the notorious +Legion of Doom could crash the national telephone network. +Fry Guy even named a date for the scheme: the Fourth of July, +a national holiday. + +This egregious example of the begging-for-arrest syndrome was shortly +followed by Fry Guy's arrest. After the Indiana telephone company figured +out who he was, the Secret Service had DNRs--Dialed Number Recorders-- +installed on his home phone lines. These devices are not taps, and can't +record the substance of phone calls, but they do record the phone numbers +of all calls going in and out. Tracing these numbers showed Fry Guy's +long-distance code fraud, his extensive ties to pirate bulletin boards, +and numerous personal calls to his LoD friends in Atlanta. By July 11, +1989, Prophet, Urvile and Leftist also had Secret Service DNR +"pen registers" installed on their own lines. + +The Secret Service showed up in force at Fry Guy's house on July 22, 1989, +to the horror of his unsuspecting parents. The raiders were led by +a special agent from the Secret Service's Indianapolis office. +However, the raiders were accompanied and advised by Timothy M. Foley +of the Secret Service's Chicago office (a gentleman about whom +we will soon be hearing a great deal). + +Following federal computer-crime techniques that had been standard +since the early 1980s, the Secret Service searched the house thoroughly, +and seized all of Fry Guy's electronic equipment and notebooks. +All Fry Guy's equipment went out the door in the custody of the +Secret Service, which put a swift end to his depredations. + +The USSS interrogated Fry Guy at length. His case was put in the charge +of Deborah Daniels, the federal US Attorney for the Southern District +of Indiana. Fry Guy was charged with eleven counts of computer fraud, +unauthorized computer access, and wire fraud. The evidence was thorough +and irrefutable. For his part, Fry Guy blamed his corruption on the +Legion of Doom and offered to testify against them. + +Fry Guy insisted that the Legion intended to crash the phone system +on a national holiday. And when AT&T crashed on Martin Luther King Day, +1990, this lent a credence to his claim that genuinely alarmed telco +security and the Secret Service. + +Fry Guy eventually pled guilty on May 31, 1990. On September 14, +he was sentenced to forty-four months' probation and four hundred hours' +community service. He could have had it much worse; but it made sense +to prosecutors to take it easy on this teenage minor, while zeroing +in on the notorious kingpins of the Legion of Doom. + +But the case against LoD had nagging flaws. Despite the best effort +of investigators, it was impossible to prove that the Legion had crashed +the phone system on January 15, because they, in fact, hadn't done so. +The investigations of 1989 did show that certain members of +the Legion of Doom had achieved unprecedented power over the telco +switching stations, and that they were in active conspiracy +to obtain more power yet. Investigators were privately convinced +that the Legion of Doom intended to do awful things with this knowledge, +but mere evil intent was not enough to put them in jail. + +And although the Atlanta Three--Prophet, Leftist, and especially Urvile-- +had taught Fry Guy plenty, they were not themselves credit-card fraudsters. +The only thing they'd "stolen" was long-distance service--and since they'd +done much of that through phone-switch manipulation, there was no easy way +to judge how much they'd "stolen," or whether this practice was even "theft" +of any easily recognizable kind. + +Fry Guy's theft of long-distance codes had cost the phone companies plenty. +The theft of long-distance service may be a fairly theoretical "loss," +but it costs genuine money and genuine time to delete all those stolen codes, +and to re-issue new codes to the innocent owners of those corrupted codes. +The owners of the codes themselves are victimized, and lose time and money +and peace of mind in the hassle. And then there were the credit-card victims +to deal with, too, and Western Union. When it came to rip-off, Fry Guy was +far more of a thief than LoD. It was only when it came to actual computer +expertise that Fry Guy was small potatoes. + +The Atlanta Legion thought most "rules" of cyberspace were for rodents +and losers, but they DID have rules. THEY NEVER CRASHED ANYTHING, +AND THEY NEVER TOOK MONEY. These were rough rules-of-thumb, and +rather dubious principles when it comes to the ethical subtleties +of cyberspace, but they enabled the Atlanta Three to operate with +a relatively clear conscience (though never with peace of mind). + +If you didn't hack for money, if you weren't robbing people of actual funds +--money in the bank, that is-- then nobody REALLY got hurt, in LoD's opinion. +"Theft of service" was a bogus issue, and "intellectual property" was +a bad joke. But LoD had only elitist contempt for rip-off artists, +"leechers," thieves. They considered themselves clean. In their opinion, +if you didn't smash-up or crash any systems --(well, not on purpose, anyhow-- +accidents can happen, just ask Robert Morris) then it was very unfair +to call you a "vandal" or a "cracker." When you were hanging out on-line +with your "pals" in telco security, you could face them down from the higher +plane of hacker morality. And you could mock the police from the supercilious +heights of your hacker's quest for pure knowledge. + +But from the point of view of law enforcement and telco security, however, +Fry Guy was not really dangerous. The Atlanta Three WERE dangerous. +It wasn't the crimes they were committing, but the DANGER, +the potential hazard, the sheer TECHNICAL POWER LoD had accumulated, +that had made the situation untenable. Fry Guy was not LoD. +He'd never laid eyes on anyone in LoD; his only contacts with them +had been electronic. Core members of the Legion of Doom tended to meet +physically for conventions every year or so, to get drunk, give each other +the hacker high-sign, send out for pizza and ravage hotel suites. +Fry Guy had never done any of this. Deborah Daniels assessed Fry Guy +accurately as "an LoD wannabe." + +Nevertheless Fry Guy's crimes would be directly attributed to LoD +in much future police propaganda. LoD would be described as +"a closely knit group" involved in "numerous illegal activities" +including "stealing and modifying individual credit histories," +and "fraudulently obtaining money and property." Fry Guy did this, +but the Atlanta Three didn't; they simply weren't into theft, +but rather intrusion. This caused a strange kink in +the prosecution's strategy. LoD were accused of +"disseminating information about attacking computers +to other computer hackers in an effort to shift the focus +of law enforcement to those other hackers and away from the Legion of Doom." + +This last accusation (taken directly from a press release by the Chicago +Computer Fraud and Abuse Task Force) sounds particularly far-fetched. +One might conclude at this point that investigators would have been +well-advised to go ahead and "shift their focus" from the "Legion of Doom." +Maybe they SHOULD concentrate on "those other hackers"--the ones who were +actually stealing money and physical objects. + +But the Hacker Crackdown of 1990 was not a simple policing action. +It wasn't meant just to walk the beat in cyberspace--it was a CRACKDOWN, +a deliberate attempt to nail the core of the operation, to send a dire +and potent message that would settle the hash of the digital underground +for good. + +By this reasoning, Fry Guy wasn't much more than the electronic equivalent +of a cheap streetcorner dope dealer. As long as the masterminds of LoD were +still flagrantly operating, pushing their mountains of illicit knowledge +right and left, and whipping up enthusiasm for blatant lawbreaking, +then there would be an INFINITE SUPPLY of Fry Guys. + +Because LoD were flagrant, they had left trails everywhere, +to be picked up by law enforcement in New York, Indiana, +Florida, Texas, Arizona, Missouri, even Australia. +But 1990's war on the Legion of Doom was led out of Illinois, +by the Chicago Computer Fraud and Abuse Task Force. + +# + +The Computer Fraud and Abuse Task Force, led by federal prosecutor +William J. Cook, had started in 1987 and had swiftly become one +of the most aggressive local "dedicated computer-crime units." +Chicago was a natural home for such a group. The world's first +computer bulletin-board system had been invented in Illinois. +The state of Illinois had some of the nation's first and sternest +computer crime laws. Illinois State Police were markedly alert +to the possibilities of white-collar crime and electronic fraud. + +And William J. Cook in particular was a rising star in +electronic crime-busting. He and his fellow federal prosecutors +at the U.S. Attorney's office in Chicago had a tight relation +with the Secret Service, especially go-getting Chicago-based agent +Timothy Foley. While Cook and his Department of Justice colleagues +plotted strategy, Foley was their man on the street. + +Throughout the 1980s, the federal government had given prosecutors +an armory of new, untried legal tools against computer crime. +Cook and his colleagues were pioneers in the use of these new statutes +in the real-life cut-and-thrust of the federal courtroom. + +On October 2, 1986, the US Senate had passed the +"Computer Fraud and Abuse Act" unanimously, but there +were pitifully few convictions under this statute. +Cook's group took their name from this statute, +since they were determined to transform this powerful but +rather theoretical Act of Congress into a real-life engine +of legal destruction against computer fraudsters and scofflaws. + +It was not a question of merely discovering crimes, +investigating them, and then trying and punishing their +perpetrators. The Chicago unit, like most everyone else +in the business, already KNEW who the bad guys were: +the Legion of Doom and the writers and editors of Phrack. +The task at hand was to find some legal means of putting +these characters away. + +This approach might seem a bit dubious, to someone not +acquainted with the gritty realities of prosecutorial work. +But prosecutors don't put people in jail for crimes +they have committed; they put people in jail for crimes +they have committed THAT CAN BE PROVED IN COURT. +Chicago federal police put Al Capone in prison +for income-tax fraud. Chicago is a big town, +with a rough-and-ready bare-knuckle tradition +on both sides of the law. + +Fry Guy had broken the case wide open and alerted telco security +to the scope of the problem. But Fry Guy's crimes would not +put the Atlanta Three behind bars--much less the wacko underground +journalists of Phrack. So on July 22, 1989, the same day that +Fry Guy was raided in Indiana, the Secret Service descended upon +the Atlanta Three. + +This was likely inevitable. By the summer of 1989, law enforcement +were closing in on the Atlanta Three from at least six directions at once. +First, there were the leads from Fry Guy, which had led to the DNR registers +being installed on the lines of the Atlanta Three. The DNR evidence alone +would have finished them off, sooner or later. + +But second, the Atlanta lads were already well-known to Control-C +and his telco security sponsors. LoD's contacts with telco security +had made them overconfident and even more boastful than usual; +they felt that they had powerful friends in high places, +and that they were being openly tolerated by telco security. +But BellSouth's Intrusion Task Force were hot on the trail of LoD +and sparing no effort or expense. + +The Atlanta Three had also been identified by name and listed +on the extensive anti-hacker files maintained, and retailed for pay, +by private security operative John Maxfield of Detroit. +Maxfield, who had extensive ties to telco security +and many informants in the underground, was a bete noire +of the Phrack crowd, and the dislike was mutual. + + +The Atlanta Three themselves had written articles for Phrack. +This boastful act could not possibly escape telco and law enforcement +attention. + +"Knightmare," a high-school age hacker from Arizona, +was a close friend and disciple of Atlanta LoD, +but he had been nabbed by the formidable Arizona +Organized Crime and Racketeering Unit. Knightmare was +on some of LoD's favorite boards--"Black Ice" in particular-- +and was privy to their secrets. And to have Gail Thackeray, +the Assistant Attorney General of Arizona, on one's trail +was a dreadful peril for any hacker. + +And perhaps worst of all, Prophet had committed a major blunder +by passing an illicitly copied BellSouth computer-file to Knight Lightning, +who had published it in Phrack. This, as we will see, was an act of dire +consequence for almost everyone concerned. + +On July 22, 1989, the Secret Service showed up at the Leftist's house, +where he lived with his parents. A massive squad of some twenty officers +surrounded the building: Secret Service, federal marshals, local police, +possibly BellSouth telco security; it was hard to tell in the crush. +Leftist's dad, at work in his basement office, first noticed +a muscular stranger in plain clothes crashing through the +back yard with a drawn pistol. As more strangers poured +into the house, Leftist's dad naturally assumed there was +an armed robbery in progress. + +Like most hacker parents, Leftist's mom and dad had only the vaguest +notions of what their son had been up to all this time. Leftist had +a day-job repairing computer hardware. His obsession with computers +seemed a bit odd, but harmless enough, and likely to produce a well- +paying career. The sudden, overwhelming raid left Leftist's +parents traumatized. + +The Leftist himself had been out after work with his co-workers, +surrounding a couple of pitchers of margaritas. As he came trucking +on tequila-numbed feet up the pavement, toting a bag full of floppy-disks, +he noticed a large number of unmarked cars parked in his driveway. +All the cars sported tiny microwave antennas. + +The Secret Service had knocked the front door off its hinges, +almost flattening his mom. + +Inside, Leftist was greeted by Special Agent James Cool +of the US Secret Service, Atlanta office. Leftist was flabbergasted. +He'd never met a Secret Service agent before. He could not imagine +that he'd ever done anything worthy of federal attention. +He'd always figured that if his activities became intolerable, +one of his contacts in telco security would give him a private +phone-call and tell him to knock it off. + +But now Leftist was pat-searched for weapons by grim professionals, +and his bag of floppies was quickly seized. He and his parents were +all shepherded into separate rooms and grilled at length as a score +of officers scoured their home for anything electronic. + +Leftist was horrified as his treasured IBM AT personal computer +with its forty-meg hard disk, and his recently purchased 80386 IBM-clone +with a whopping hundred-meg hard disk, both went swiftly out the door +in Secret Service custody. They also seized all his disks, all his notebooks, +and a tremendous booty in dogeared telco documents that Leftist had snitched +out of trash dumpsters. + +Leftist figured the whole thing for a big misunderstanding. +He'd never been into MILITARY computers. He wasn't a SPY or a COMMUNIST. +He was just a good ol' Georgia hacker, and now he just wanted all these +people out of the house. But it seemed they wouldn't go until he made +some kind of statement. + +And so, he levelled with them. + +And that, Leftist said later from his federal prison camp in Talladega, +Alabama, was a big mistake. The Atlanta area was unique, +in that it had three members of the Legion of Doom who actually +occupied more or less the same physical locality. Unlike the rest +of LoD, who tended to associate by phone and computer, +Atlanta LoD actually WERE "tightly knit." It was no real +surprise that the Secret Service agents apprehending Urvile +at the computer-labs at Georgia Tech, would discover Prophet +with him as well. + +Urvile, a 21-year-old Georgia Tech student in polymer chemistry, +posed quite a puzzling case for law enforcement. Urvile--also known +as "Necron 99," as well as other handles, for he tended to change his +cover-alias about once a month--was both an accomplished hacker +and a fanatic simulation-gamer. + +Simulation games are an unusual hobby; but then hackers are unusual people, +and their favorite pastimes tend to be somewhat out of the ordinary. +The best-known American simulation game is probably "Dungeons & Dragons," +a multi-player parlor entertainment played with paper, maps, pencils, +statistical tables and a variety of oddly-shaped dice. Players pretend +to be heroic characters exploring a wholly-invented fantasy world. +The fantasy worlds of simulation gaming are commonly pseudo-medieval, +involving swords and sorcery--spell-casting wizards, knights in armor, +unicorns and dragons, demons and goblins. + +Urvile and his fellow gamers preferred their fantasies highly technological. +They made use of a game known as "G.U.R.P.S.," the "Generic Universal Role +Playing System," published by a company called Steve Jackson Games (SJG). + +"G.U.R.P.S." served as a framework for creating a wide variety of artificial +fantasy worlds. Steve Jackson Games published a smorgasboard of books, +full of detailed information and gaming hints, which were used to flesh-out +many different fantastic backgrounds for the basic GURPS framework. +Urvile made extensive use of two SJG books called GURPS High-Tech +and GURPS Special Ops. + +In the artificial fantasy-world of GURPS Special Ops, +players entered a modern fantasy of intrigue and international espionage. +On beginning the game, players started small and powerless, +perhaps as minor-league CIA agents or penny-ante arms dealers. +But as players persisted through a series of game sessions +(game sessions generally lasted for hours, over long, +elaborate campaigns that might be pursued for months on end) +then they would achieve new skills, new knowledge, new power. +They would acquire and hone new abilities, such as marksmanship, +karate, wiretapping, or Watergate burglary. They could also win +various kinds of imaginary booty, like Berettas, or martini shakers, +or fast cars with ejection seats and machine-guns under the headlights. + +As might be imagined from the complexity of these games, +Urvile's gaming notes were very detailed and extensive. +Urvile was a "dungeon-master," inventing scenarios +for his fellow gamers, giant simulated adventure-puzzles +for his friends to unravel. Urvile's game notes covered +dozens of pages with all sorts of exotic lunacy, all about +ninja raids on Libya and break-ins on encrypted Red Chinese supercomputers. +His notes were written on scrap-paper and kept in loose-leaf binders. + +The handiest scrap paper around Urvile's college digs were the many pounds of +BellSouth printouts and documents that he had snitched out of telco dumpsters. +His notes were written on the back of misappropriated telco property. +Worse yet, the gaming notes were chaotically interspersed with Urvile's +hand-scrawled records involving ACTUAL COMPUTER INTRUSIONS that he +had committed. + +Not only was it next to impossible to tell Urvile's fantasy game-notes +from cyberspace "reality," but Urvile himself barely made this distinction. +It's no exaggeration to say that to Urvile it was ALL a game. Urvile was +very bright, highly imaginative, and quite careless of other people's notions +of propriety. His connection to "reality" was not something to which he paid +a great deal of attention. + +Hacking was a game for Urvile. It was an amusement he was carrying out, +it was something he was doing for fun. And Urvile was an obsessive young man. +He could no more stop hacking than he could stop in the middle of +a jigsaw puzzle, or stop in the middle of reading a Stephen Donaldson +fantasy trilogy. (The name "Urvile" came from a best-selling Donaldson novel.) + +Urvile's airy, bulletproof attitude seriously annoyed his interrogators. +First of all, he didn't consider that he'd done anything wrong. +There was scarcely a shred of honest remorse in him. On the contrary, +he seemed privately convinced that his police interrogators were operating +in a demented fantasy-world all their own. Urvile was too polite +and well-behaved to say this straight-out, but his reactions were askew +and disquieting. + +For instance, there was the business about LoD's ability +to monitor phone-calls to the police and Secret Service. +Urvile agreed that this was quite possible, and posed +no big problem for LoD. In fact, he and his friends +had kicked the idea around on the "Black Ice" board, +much as they had discussed many other nifty notions, +such as building personal flame-throwers and jury-rigging +fistfulls of blasting-caps. They had hundreds of dial-up numbers +for government agencies that they'd gotten through scanning Atlanta phones, +or had pulled from raided VAX/VMS mainframe computers. + +Basically, they'd never gotten around to listening in on the cops +because the idea wasn't interesting enough to bother with. +Besides, if they'd been monitoring Secret Service phone calls, +obviously they'd never have been caught in the first place. Right? + +The Secret Service was less than satisfied with this rapier-like hacker logic. + +Then there was the issue of crashing the phone system. No problem, +Urvile admitted sunnily. Atlanta LoD could have shut down phone service +all over Atlanta any time they liked. EVEN THE 911 SERVICE? +Nothing special about that, Urvile explained patiently. +Bring the switch to its knees, with say the UNIX "makedir" bug, +and 911 goes down too as a matter of course. The 911 system +wasn't very interesting, frankly. It might be tremendously +interesting to cops (for odd reasons of their own), but as +technical challenges went, the 911 service was yawnsville. + +So of course the Atlanta Three could crash service. +They probably could have crashed service all over +BellSouth territory, if they'd worked at it for a while. +But Atlanta LoD weren't crashers. Only losers and rodents +were crashers. LoD were ELITE. + +Urvile was privately convinced that sheer technical +expertise could win him free of any kind of problem. +As far as he was concerned, elite status in the digital +underground had placed him permanently beyond the intellectual +grasp of cops and straights. Urvile had a lot to learn. + +Of the three LoD stalwarts, Prophet was in the most direct trouble. +Prophet was a UNIX programming expert who burrowed in and out +of the Internet as a matter of course. He'd started his hacking +career at around age 14, meddling with a UNIX mainframe system +at the University of North Carolina. + +Prophet himself had written the handy Legion of Doom +file "UNIX Use and Security From the Ground Up." +UNIX (pronounced "you-nicks") is a powerful, +flexible computer operating-system, for multi-user, +multi-tasking computers. In 1969, when UNIX was created +in Bell Labs, such computers were exclusive to large +corporations and universities, but today UNIX is run +on thousands of powerful home machines. UNIX was +particularly well-suited to telecommunications programming, +and had become a standard in the field. Naturally, UNIX +also became a standard for the elite hacker and phone phreak. +Lately, Prophet had not been so active as Leftist and Urvile, +but Prophet was a recidivist. In 1986, when he was eighteen, +Prophet had been convicted of "unauthorized access +to a computer network" in North Carolina. He'd been +discovered breaking into the Southern Bell Data Network, +a UNIX-based internal telco network supposedly closed to the public. +He'd gotten a typical hacker sentence: six months suspended, +120 hours community service, and three years' probation. + +After that humiliating bust, Prophet had gotten rid of most of his +tonnage of illicit phreak and hacker data, and had tried to go straight. +He was, after all, still on probation. But by the autumn of 1988, +the temptations of cyberspace had proved too much for young Prophet, +and he was shoulder-to-shoulder with Urvile and Leftist into some +of the hairiest systems around. + +In early September 1988, he'd broken into BellSouth's centralized +automation system, AIMSX or "Advanced Information Management System." +AIMSX was an internal business network for BellSouth, where telco +employees stored electronic mail, databases, memos, and calendars, +and did text processing. Since AIMSX did not have public dial-ups, +it was considered utterly invisible to the public, and was not well-secured +--it didn't even require passwords. Prophet abused an account known +as "waa1," the personal account of an unsuspecting telco employee. +Disguised as the owner of waa1, Prophet made about ten visits to AIMSX. + +Prophet did not damage or delete anything in the system. +His presence in AIMSX was harmless and almost invisible. +But he could not rest content with that. + +One particular piece of processed text on AIMSX was a telco document +known as "Bell South Standard Practice 660-225-104SV Control Office +Administration of Enhanced 911 Services for Special Services +and Major Account Centers dated March 1988." + +Prophet had not been looking for this document. It was merely one +among hundreds of similar documents with impenetrable titles. +However, having blundered over it in the course of his illicit +wanderings through AIMSX, he decided to take it with him as a trophy. +It might prove very useful in some future boasting, bragging, +and strutting session. So, some time in September 1988, +Prophet ordered the AIMSX mainframe computer to copy this document +(henceforth called simply called "the E911 Document") and to transfer +this copy to his home computer. + +No one noticed that Prophet had done this. He had "stolen" +the E911 Document in some sense, but notions of property +in cyberspace can be tricky. BellSouth noticed nothing wrong, +because BellSouth still had their original copy. They had not +been "robbed" of the document itself. Many people were supposed +to copy this document--specifically, people who worked for the +nineteen BellSouth "special services and major account centers," +scattered throughout the Southeastern United States. That was +what it was for, why it was present on a computer network +in the first place: so that it could be copied and read-- +by telco employees. But now the data had been copied +by someone who wasn't supposed to look at it. + +Prophet now had his trophy. But he further decided to store +yet another copy of the E911 Document on another person's computer. +This unwitting person was a computer enthusiast named Richard Andrews +who lived near Joliet, Illinois. Richard Andrews was a UNIX programmer +by trade, and ran a powerful UNIX board called "Jolnet," in the basement +of his house. + +Prophet, using the handle "Robert Johnson," had obtained an account +on Richard Andrews' computer. And there he stashed the E911 Document, +by storing it in his own private section of Andrews' computer. + +Why did Prophet do this? If Prophet had eliminated the E911 Document +from his own computer, and kept it hundreds of miles away, on another machine, under an +alias, then he might have been fairly safe from discovery and prosecution-- +although his sneaky action had certainly put the unsuspecting Richard Andrews +at risk. + +But, like most hackers, Prophet was a pack-rat for illicit data. +When it came to the crunch, he could not bear to part from his trophy. +When Prophet's place in Decatur, Georgia was raided in July 1989, +there was the E911 Document, a smoking gun. And there was Prophet +in the hands of the Secret Service, doing his best to "explain." + +Our story now takes us away from the Atlanta Three and their raids +of the Summer of 1989. We must leave Atlanta Three "cooperating fully" +with their numerous investigators. And all three of them did cooperate, +as their Sentencing Memorandum from the US District Court of the +Northern Division of Georgia explained--just before all three of them +were sentenced to various federal prisons in November 1990. + +We must now catch up on the other aspects of the war on the Legion of Doom. +The war on the Legion was a war on a network--in fact, a network of three +networks, which intertwined and interrelated in a complex fashion. +The Legion itself, with Atlanta LoD, and their hanger-on Fry Guy, +were the first network. The second network was Phrack magazine, +with its editors and contributors. + +The third network involved the electronic circle around a hacker +known as "Terminus." + +The war against these hacker networks was carried out by +a law enforcement network. Atlanta LoD and Fry Guy +were pursued by USSS agents and federal prosecutors in Atlanta, +Indiana, and Chicago. "Terminus" found himself pursued by USSS +and federal prosecutors from Baltimore and Chicago. And the war +against Phrack was almost entirely a Chicago operation. + +The investigation of Terminus involved a great deal of energy, +mostly from the Chicago Task Force, but it was to be the least-known +and least-publicized of the Crackdown operations. Terminus, who lived +in Maryland, was a UNIX programmer and consultant, fairly well-known +(under his given name) in the UNIX community, as an acknowledged expert +on AT&T minicomputers. Terminus idolized AT&T, especially Bellcore, +and longed for public recognition as a UNIX expert; his highest ambition +was to work for Bell Labs. + +But Terminus had odd friends and a spotted history. +Terminus had once been the subject of an admiring interview +in Phrack (Volume II, Issue 14, Phile 2--dated May 1987). +In this article, Phrack co-editor Taran King described +"Terminus" as an electronics engineer, 5'9", brown-haired, +born in 1959--at 28 years old, quite mature for a hacker. + +Terminus had once been sysop of a phreak/hack underground board +called "MetroNet," which ran on an Apple II. Later he'd replaced +"MetroNet" with an underground board called "MegaNet," +specializing in IBMs. In his younger days, Terminus had written +one of the very first and most elegant code-scanning programs +for the IBM-PC. This program had been widely distributed +in the underground. Uncounted legions of PC-owning phreaks and +hackers had used Terminus's scanner program to rip-off telco codes. +This feat had not escaped the attention of telco security; +it hardly could, since Terminus's earlier handle, "Terminal Technician," +was proudly written right on the program. + +When he became a full-time computer professional +(specializing in telecommunications programming), +he adopted the handle Terminus, meant to indicate that he +had "reached the final point of being a proficient hacker." +He'd moved up to the UNIX-based "Netsys" board on an AT&T computer, +with four phone lines and an impressive 240 megs of storage. +"Netsys" carried complete issues of Phrack, and Terminus was +quite friendly with its publishers, Taran King and Knight Lightning. + +In the early 1980s, Terminus had been a regular on Plovernet, +Pirate-80, Sherwood Forest and Shadowland, all well-known pirate boards, +all heavily frequented by the Legion of Doom. As it happened, Terminus +was never officially "in LoD," because he'd never been given the official +LoD high-sign and back-slap by Legion maven Lex Luthor. Terminus had +never physically met anyone from LoD. But that scarcely mattered much-- +the Atlanta Three themselves had never been officially vetted by Lex, either. + +As far as law enforcement was concerned, the issues were clear. +Terminus was a full-time, adult computer professional +with particular skills at AT&T software and hardware-- +but Terminus reeked of the Legion of Doom and the underground. + +On February 1, 1990--half a month after the Martin Luther King Day Crash-- +USSS agents Tim Foley from Chicago, and Jack Lewis from the Baltimore office, +accompanied by AT&T security officer Jerry Dalton, travelled to Middle Town, +Maryland. There they grilled Terminus in his home (to the stark terror of +his wife and small children), and, in their customary fashion, hauled his +computers out the door. + +The Netsys machine proved to contain a plethora of arcane UNIX software-- +proprietary source code formally owned by AT&T. Software such as: +UNIX System Five Release 3.2; UNIX SV Release 3.1; UUCP communications +software; KORN SHELL; RFS; IWB; WWB; DWB; the C++ programming language; +PMON; TOOL CHEST; QUEST; DACT, and S FIND. + +In the long-established piratical tradition of the underground, +Terminus had been trading this illicitly-copied software with +a small circle of fellow UNIX programmers. Very unwisely, +he had stored seven years of his electronic mail on his Netsys machine, +which documented all the friendly arrangements he had made with +his various colleagues. + +Terminus had not crashed the AT&T phone system on January 15. +He was, however, blithely running a not-for-profit AT&T +software-piracy ring. This was not an activity AT&T found amusing. +AT&T security officer Jerry Dalton valued this "stolen" property +at over three hundred thousand dollars. + +AT&T's entry into the tussle of free enterprise had been complicated +by the new, vague groundrules of the information economy. +Until the break-up of Ma Bell, AT&T was forbidden to sell +computer hardware or software. Ma Bell was the phone company; +Ma Bell was not allowed to use the enormous revenue from +telephone utilities, in order to finance any entry into +the computer market. + +AT&T nevertheless invented the UNIX operating system. +And somehow AT&T managed to make UNIX a minor source of income. +Weirdly, UNIX was not sold as computer software, +but actually retailed under an obscure regulatory +exemption allowing sales of surplus equipment and scrap. +Any bolder attempt to promote or retail UNIX would have +aroused angry legal opposition from computer companies. +Instead, UNIX was licensed to universities, at modest rates, +where the acids of academic freedom ate away steadily at AT&T's +proprietary rights. + +Come the breakup, AT&T recognized that UNIX was a potential gold-mine. +By now, large chunks of UNIX code had been created that were not AT&T's, +and were being sold by others. An entire rival UNIX-based operating system +had arisen in Berkeley, California (one of the world's great founts of +ideological hackerdom). Today, "hackers" commonly consider "Berkeley UNIX" +to be technically superior to AT&T's "System V UNIX," but AT&T has not +allowed mere technical elegance to intrude on the real-world business +of marketing proprietary software. AT&T has made its own code deliberately +incompatible with other folks' UNIX, and has written code that it can prove +is copyrightable, even if that code happens to be somewhat awkward--"kludgey." +AT&T UNIX user licenses are serious business agreements, replete with very +clear copyright statements and non-disclosure clauses. + +AT&T has not exactly kept the UNIX cat in the bag, +but it kept a grip on its scruff with some success. +By the rampant, explosive standards of software piracy, +AT&T UNIX source code is heavily copyrighted, well-guarded, +well-licensed. UNIX was traditionally run only on +mainframe machines, owned by large groups of suit-and-tie +professionals, rather than on bedroom machines where +people can get up to easy mischief. + +And AT&T UNIX source code is serious high-level programming. +The number of skilled UNIX programmers with any actual motive +to swipe UNIX source code is small. It's tiny, compared to +the tens of thousands prepared to rip-off, say, entertaining +PC games like "Leisure Suit Larry." + +But by 1989, the warez-d00d underground, in the persons of Terminus +and his friends, was gnawing at AT&T UNIX. And the property in question +was not sold for twenty bucks over the counter at the local branch of +Babbage's or Egghead's; this was massive, sophisticated, multi-line, +multi-author corporate code worth tens of thousands of dollars. + +It must be recognized at this point that Terminus's purported ring of UNIX +software pirates had not actually made any money from their suspected crimes. +The $300,000 dollar figure bandied about for the contents of Terminus's +computer did not mean that Terminus was in actual illicit possession +of three hundred thousand of AT&T's dollars. Terminus was shipping +software back and forth, privately, person to person, for free. +He was not making a commercial business of piracy. He hadn't +asked for money; he didn't take money. He lived quite modestly. + +AT&T employees--as well as freelance UNIX consultants, like Terminus-- +commonly worked with "proprietary" AT&T software, both in the office +and at home on their private machines. AT&T rarely sent security officers +out to comb the hard disks of its consultants. Cheap freelance UNIX +contractors were quite useful to AT&T; they didn't have health insurance +or retirement programs, much less union membership in the Communication +Workers of America. They were humble digital drudges, wandering with mop +and bucket through the Great Technological Temple of AT&T; but when the +Secret Service arrived at their homes, it seemed they were eating with +company silverware and sleeping on company sheets! Outrageously, they +behaved as if the things they worked with every day belonged to them! + +And these were no mere hacker teenagers with their hands full +of trash-paper and their noses pressed to the corporate windowpane. +These guys were UNIX wizards, not only carrying AT&T data in their +machines and their heads, but eagerly networking about it, +over machines that were far more powerful than anything previously +imagined in private hands. How do you keep people disposable, +yet assure their awestruck respect for your property? It was a dilemma. + +Much UNIX code was public-domain, available for free. Much "proprietary" +UNIX code had been extensively re-written, perhaps altered so much that it +became an entirely new product--or perhaps not. Intellectual property rights +for software developers were, and are, extraordinarily complex and confused. +And software "piracy," like the private copying of videos, is one of the most +widely practiced "crimes" in the world today. + +The USSS were not experts in UNIX or familiar with the customs of its use. +The United States Secret Service, considered as a body, did not have one single +person in it who could program in a UNIX environment--no, not even one. +The Secret Service WERE making extensive use of expert help, but the "experts" +they had chosen were AT&T and Bellcore security officials, the very victims of +the purported crimes under investigation, the very people whose interest in +AT&T's "proprietary" software was most pronounced. + +On February 6, 1990, Terminus was arrested by Agent Lewis. +Eventually, Terminus would be sent to prison for his illicit +use of a piece of AT&T software. + +The issue of pirated AT&T software would bubble along in the background +during the war on the Legion of Doom. Some half-dozen of Terminus's on-line +acquaintances, including people in Illinois, Texas and California, +were grilled by the Secret Service in connection with the illicit +copying of software. Except for Terminus, however, none were charged +with a crime. None of them shared his peculiar prominence in the +hacker underground. + +But that did not mean that these people would, or could, +stay out of trouble. The transferral of illicit data in +cyberspace is hazy and ill-defined business, with paradoxical +dangers for everyone concerned: hackers, signal carriers, +board owners, cops, prosecutors, even random passers-by. +Sometimes, well-meant attempts to avert trouble +or punish wrongdoing bring more trouble than +would simple ignorance, indifference or impropriety. + +Terminus's "Netsys" board was not a common-or-garden +bulletin board system, though it had most of the usual +functions of a board. Netsys was not a stand-alone machine, +but part of the globe-spanning "UUCP" cooperative network. +The UUCP network uses a set of Unix software programs called +"Unix-to-Unix Copy," which allows Unix systems to throw data to +one another at high speed through the public telephone network. +UUCP is a radically decentralized, not-for-profit network of UNIX computers. +There are tens of thousands of these UNIX machines. Some are small, +but many are powerful and also link to other networks. UUCP has +certain arcane links to major networks such as JANET, EasyNet, BITNET, +JUNET, VNET, DASnet, PeaceNet and FidoNet, as well as the gigantic Internet. +(The so-called "Internet" is not actually a network itself, but rather an +"internetwork" connections standard that allows several globe-spanning +computer networks to communicate with one another. Readers fascinated +by the weird and intricate tangles of modern computer networks may enjoy +John S. Quarterman's authoritative 719-page explication, The Matrix, +Digital Press, 1990.) + +A skilled user of Terminus' UNIX machine could send and receive +electronic mail from almost any major computer network in the world. +Netsys was not called a "board" per se, but rather a "node." +"Nodes" were larger, faster, and more sophisticated than mere "boards," +and for hackers, to hang out on internationally-connected "nodes" +was quite the step up from merely hanging out on local "boards." + +Terminus's Netsys node in Maryland had a number of direct +links to other, similar UUCP nodes, run by people who shared his +interests and at least something of his free-wheeling attitude. +One of these nodes was Jolnet, owned by Richard Andrews, who, +like Terminus, was an independent UNIX consultant. +Jolnet also ran UNIX, and could be contacted at high speed +by mainframe machines from all over the world. Jolnet was +quite a sophisticated piece of work, technically speaking, +but it was still run by an individual, as a private, +not-for-profit hobby. Jolnet was mostly used by other +UNIX programmers--for mail, storage, and access to networks. +Jolnet supplied access network access to about two hundred people, +as well as a local junior college. + +Among its various features and services, Jolnet also carried +Phrack magazine. + +For reasons of his own, Richard Andrews had become suspicious +of a new user called "Robert Johnson." Richard Andrews +took it upon himself to have a look at what "Robert Johnson" +was storing in Jolnet. And Andrews found the E911 Document. + +"Robert Johnson" was the Prophet from the Legion of Doom, +and the E911 Document was illicitly copied data from Prophet's +raid on the BellSouth computers. + +The E911 Document, a particularly illicit piece of digital property, +was about to resume its long, complex, and disastrous career. + +It struck Andrews as fishy that someone not a telephone employee +should have a document referring to the "Enhanced 911 System." +Besides, the document itself bore an obvious warning. + +"WARNING: NOT FOR USE OR DISCLOSURE OUTSIDE BELLSOUTH +OR ANY OF ITS SUBSIDIARIES EXCEPT UNDER WRITTEN AGREEMENT." + +These standard nondisclosure tags are often appended to all sorts +of corporate material. Telcos as a species are particularly notorious +for stamping most everything in sight as "not for use or disclosure." +Still, this particular piece of data was about the 911 System. +That sounded bad to Rich Andrews. + +Andrews was not prepared to ignore this sort of trouble. +He thought it would be wise to pass the document along +to a friend and acquaintance on the UNIX network, for consultation. +So, around September 1988, Andrews sent yet another copy of the +E911 Document electronically to an AT&T employee, one Charles Boykin, +who ran a UNIX-based node called "attctc" in Dallas, Texas. + +"Attctc" was the property of AT&T, and was run from AT&T's +Customer Technology Center in Dallas, hence the name "attctc." +"Attctc" was better-known as "Killer," the name of the machine +that the system was running on. "Killer" was a hefty, powerful, +AT&T 3B2 500 model, a multi-user, multi-tasking UNIX platform +with 32 meg of memory and a mind-boggling 3.2 Gigabytes of storage. +When Killer had first arrived in Texas, in 1985, the 3B2 had been +one of AT&T's great white hopes for going head-to-head with IBM +for the corporate computer-hardware market. "Killer" had been shipped +to the Customer Technology Center in the Dallas Infomart, essentially +a high-technology mall, and there it sat, a demonstration model. + +Charles Boykin, a veteran AT&T hardware and digital communications expert, +was a local technical backup man for the AT&T 3B2 system. As a display model +in the Infomart mall, "Killer" had little to do, and it seemed a shame +to waste the system's capacity. So Boykin ingeniously wrote some UNIX +bulletin-board software for "Killer," and plugged the machine in to the +local phone network. "Killer's" debut in late 1985 made it the first +publicly available UNIX site in the state of Texas. Anyone who wanted to +play was welcome. + +The machine immediately attracted an electronic community. +It joined the UUCP network, and offered network links +to over eighty other computer sites, all of which became dependent +on Killer for their links to the greater world of cyberspace. +And it wasn't just for the big guys; personal computer users +also stored freeware programs for the Amiga, the Apple, +the IBM and the Macintosh on Killer's vast 3,200 meg archives. +At one time, Killer had the largest library of public-domain +Macintosh software in Texas. + +Eventually, Killer attracted about 1,500 users, +all busily communicating, uploading and downloading, +getting mail, gossipping, and linking to arcane +and distant networks. + +Boykin received no pay for running Killer. He considered +it good publicity for the AT&T 3B2 system (whose sales were +somewhat less than stellar), but he also simply enjoyed +the vibrant community his skill had created. He gave away +the bulletin-board UNIX software he had written, free of charge. + +In the UNIX programming community, Charlie Boykin had the +reputation of a warm, open-hearted, level-headed kind of guy. +In 1989, a group of Texan UNIX professionals voted Boykin +"System Administrator of the Year." He was considered +a fellow you could trust for good advice. + +In September 1988, without warning, the E911 Document +came plunging into Boykin's life, forwarded by Richard Andrews. +Boykin immediately recognized that the Document was hot property. +He was not a voice-communications man, and knew little about +the ins and outs of the Baby Bells, but he certainly knew what +the 911 System was, and he was angry to see confidential data +about it in the hands of a nogoodnik. This was clearly a +matter for telco security. So, on September 21, 1988, Boykin +made yet ANOTHER copy of the E911 Document and passed this +one along to a professional acquaintance of his, one Jerome Dalton, +from AT&T Corporate Information Security. Jerry Dalton was the +very fellow who would later raid Terminus's house. + +From AT&T's security division, the E911 Document went to Bellcore. + +Bellcore (or BELL COmmunications REsearch) had once been the central +laboratory of the Bell System. Bell Labs employees had invented +the UNIX operating system. Now Bellcore was a quasi-independent, +jointly owned company that acted as the research arm for all seven +of the Baby Bell RBOCs. Bellcore was in a good position to co-ordinate +security technology and consultation for the RBOCs, and the gentleman in +charge of this effort was Henry M. Kluepfel, a veteran of the Bell System +who had worked there for twenty-four years. + +On October 13, 1988, Dalton passed the E911 Document to Henry Kluepfel. +Kluepfel, a veteran expert witness in telecommunications fraud and +computer-fraud cases, had certainly seen worse trouble than this. +He recognized the document for what it was: a trophy from a hacker break-in. + +However, whatever harm had been done in the intrusion was presumably old news. +At this point there seemed little to be done. Kluepfel made a careful note +of the circumstances and shelved the problem for the time being. + +Whole months passed. + +February 1989 arrived. The Atlanta Three were living it up +in Bell South's switches, and had not yet met their comeuppance. +The Legion was thriving. So was Phrack magazine. +A good six months had passed since Prophet's AIMSX break-in. +Prophet, as hackers will, grew weary of sitting on his laurels. +"Knight Lightning" and "Taran King," the editors of Phrack, +were always begging Prophet for material they could publish. +Prophet decided that the heat must be off by this time, +and that he could safely brag, boast, and strut. + +So he sent a copy of the E911 Document--yet another one-- +from Rich Andrews' Jolnet machine to Knight Lightning's +BITnet account at the University of Missouri. +Let's review the fate of the document so far. + +0. The original E911 Document. This in the AIMSX system +on a mainframe computer in Atlanta, available to hundreds of people, +but all of them, presumably, BellSouth employees. An unknown number +of them may have their own copies of this document, but they are all +professionals and all trusted by the phone company. + +1. Prophet's illicit copy, at home on his own computer in Decatur, Georgia. + +2. Prophet's back-up copy, stored on Rich Andrew's Jolnet machine + in the basement of Rich Andrews' house near Joliet Illinois. + +3. Charles Boykin's copy on "Killer" in Dallas, Texas, + sent by Rich Andrews from Joliet. + +4. Jerry Dalton's copy at AT&T Corporate Information Security in New Jersey, + sent from Charles Boykin in Dallas. + +5. Henry Kluepfel's copy at Bellcore security headquarters in New Jersey, + sent by Dalton. +6. Knight Lightning's copy, sent by Prophet from Rich Andrews' machine, + and now in Columbia, Missouri. + +We can see that the "security" situation of this proprietary document, +once dug out of AIMSX, swiftly became bizarre. Without any money +changing hands, without any particular special effort, this data +had been reproduced at least six times and had spread itself all over +the continent. By far the worst, however, was yet to come. + +In February 1989, Prophet and Knight Lightning bargained electronically +over the fate of this trophy. Prophet wanted to boast, but, at the same time, +scarcely wanted to be caught. + +For his part, Knight Lightning was eager to publish as much of the document +as he could manage. Knight Lightning was a fledgling political-science major +with a particular interest in freedom-of-information issues. He would gladly +publish most anything that would reflect glory on the prowess of the +underground and embarrass the telcos. However, Knight Lightning himself +had contacts in telco security, and sometimes consulted them on material +he'd received that might be too dicey for publication. + +Prophet and Knight Lightning decided to edit the E911 Document +so as to delete most of its identifying traits. First of all, +its large "NOT FOR USE OR DISCLOSURE" warning had to go. +Then there were other matters. For instance, it listed +the office telephone numbers of several BellSouth 911 +specialists in Florida. If these phone numbers were +published in Phrack, the BellSouth employees involved +would very likely be hassled by phone phreaks, +which would anger BellSouth no end, and pose a +definite operational hazard for both Prophet and Phrack. + +So Knight Lightning cut the Document almost in half, +removing the phone numbers and some of the touchier +and more specific information. He passed it back +electronically to Prophet; Prophet was still nervous, +so Knight Lightning cut a bit more. They finally agreed +that it was ready to go, and that it would be published +in Phrack under the pseudonym, "The Eavesdropper." + +And this was done on February 25, 1989. + +The twenty-fourth issue of Phrack featured a chatty interview +with co-ed phone-phreak "Chanda Leir," three articles on BITNET +and its links to other computer networks, an article on 800 and 900 +numbers by "Unknown User," "VaxCat's" article on telco basics +(slyly entitled "Lifting Ma Bell's Veil of Secrecy,)" and +the usual "Phrack World News." + +The News section, with painful irony, featured an extended account +of the sentencing of "Shadowhawk," an eighteen-year-old Chicago hacker +who had just been put in federal prison by William J. Cook himself. + +And then there were the two articles by "The Eavesdropper." +The first was the edited E911 Document, now titled +"Control Office Administration Of Enhanced 911 Services +for Special Services and Major Account Centers." +Eavesdropper's second article was a glossary of terms +explaining the blizzard of telco acronyms and buzzwords +in the E911 Document. + +The hapless document was now distributed, in the usual Phrack routine, +to a good one hundred and fifty sites. Not a hundred and fifty PEOPLE, +mind you--a hundred and fifty SITES, some of these sites linked to UNIX +nodes or bulletin board systems, which themselves had readerships of tens, +dozens, even hundreds of people. + +This was February 1989. Nothing happened immediately. +Summer came, and the Atlanta crew were raided by the Secret Service. +Fry Guy was apprehended. Still nothing whatever happened to Phrack. +Six more issues of Phrack came out, 30 in all, more or less on +a monthly schedule. Knight Lightning and co-editor Taran King +went untouched. + +Phrack tended to duck and cover whenever the heat came down. +During the summer busts of 1987--(hacker busts tended to cluster in summer, +perhaps because hackers were easier to find at home than in college)-- +Phrack had ceased publication for several months, and laid low. +Several LoD hangers-on had been arrested, but nothing had happened +to the Phrack crew, the premiere gossips of the underground. +In 1988, Phrack had been taken over by a new editor, +"Crimson Death," a raucous youngster with a taste for anarchy files. +1989, however, looked like a bounty year for the underground. +Knight Lightning and his co-editor Taran King took up the reins again, +and Phrack flourished throughout 1989. Atlanta LoD went down hard in +the summer of 1989, but Phrack rolled merrily on. Prophet's E911 Document +seemed unlikely to cause Phrack any trouble. By January 1990, +it had been available in Phrack for almost a year. Kluepfel and Dalton, +officers of Bellcore and AT&T security, had possessed the document +for sixteen months--in fact, they'd had it even before Knight Lightning +himself, and had done nothing in particular to stop its distribution. +They hadn't even told Rich Andrews or Charles Boykin to erase the copies +from their UNIX nodes, Jolnet and Killer. + +But then came the monster Martin Luther King Day Crash of January 15, 1990. + +A flat three days later, on January 18, four agents showed up +at Knight Lightning's fraternity house. One was Timothy Foley, +the second Barbara Golden, both of them Secret Service agents +from the Chicago office. Also along was a University of Missouri +security officer, and Reed Newlin, a security man from Southwestern Bell, +the RBOC having jurisdiction over Missouri. + +Foley accused Knight Lightning of causing the nationwide crash +of the phone system. + +Knight Lightning was aghast at this allegation. On the face of it, +the suspicion was not entirely implausible--though Knight Lightning +knew that he himself hadn't done it. Plenty of hot-dog hackers +had bragged that they could crash the phone system, however. +"Shadowhawk," for instance, the Chicago hacker whom William Cook +had recently put in jail, had several times boasted on boards +that he could "shut down AT&T's public switched network." + +And now this event, or something that looked just like it, +had actually taken place. The Crash had lit a fire under +the Chicago Task Force. And the former fence-sitters at +Bellcore and AT&T were now ready to roll. The consensus +among telco security--already horrified by the skill of +the BellSouth intruders --was that the digital underground +was out of hand. LoD and Phrack must go. And in publishing +Prophet's E911 Document, Phrack had provided law enforcement +with what appeared to be a powerful legal weapon. + +Foley confronted Knight Lightning about the E911 Document. + +Knight Lightning was cowed. He immediately began "cooperating fully" +in the usual tradition of the digital underground. + +He gave Foley a complete run of Phrack, printed out in a set +of three-ring binders. He handed over his electronic mailing list +of Phrack subscribers. Knight Lightning was grilled for four hours +by Foley and his cohorts. Knight Lightning admitted that Prophet +had passed him the E911 Document, and he admitted that he had known +it was stolen booty from a hacker raid on a telephone company. +Knight Lightning signed a statement to this effect, and agreed, +in writing, to cooperate with investigators. + +Next day--January 19, 1990, a Friday --the Secret Service returned +with a search warrant, and thoroughly searched Knight Lightning's +upstairs room in the fraternity house. They took all his floppy disks, +though, interestingly, they left Knight Lightning in possession +of both his computer and his modem. (The computer had no hard disk, +and in Foley's judgement was not a store of evidence.) But this was a +very minor bright spot among Knight Lightning's rapidly multiplying troubles. +By this time, Knight Lightning was in plenty of hot water, not only with +federal police, prosecutors, telco investigators, and university security, +but with the elders of his own campus fraternity, who were outraged +to think that they had been unwittingly harboring a federal computer-criminal. + +On Monday, Knight Lightning was summoned to Chicago, where he was +further grilled by Foley and USSS veteran agent Barbara Golden, this time +with an attorney present. And on Tuesday, he was formally indicted +by a federal grand jury. + +The trial of Knight Lightning, which occurred on July 24-27, 1990, +was the crucial show-trial of the Hacker Crackdown. We will examine +the trial at some length in Part Four of this book. + +In the meantime, we must continue our dogged pursuit of the E911 Document. + +It must have been clear by January 1990 that the E911 Document, +in the form Phrack had published it back in February 1989, +had gone off at the speed of light in at least a hundred +and fifty different directions. To attempt to put this +electronic genie back in the bottle was flatly impossible. + +And yet, the E911 Document was STILL stolen property, +formally and legally speaking. Any electronic transference +of this document, by anyone unauthorized to have it, +could be interpreted as an act of wire fraud. Interstate +transfer of stolen property, including electronic property, +was a federal crime. + +The Chicago Computer Fraud and Abuse Task Force had been assured +that the E911 Document was worth a hefty sum of money. In fact, +they had a precise estimate of its worth from BellSouth security personnel: +$79,449. A sum of this scale seemed to warrant vigorous prosecution. +Even if the damage could not be undone, at least this large sum +offered a good legal pretext for stern punishment of the thieves. +It seemed likely to impress judges and juries. And it could be used +in court to mop up the Legion of Doom. + +The Atlanta crowd was already in the bag, by the time +the Chicago Task Force had gotten around to Phrack. +But the Legion was a hydra-headed thing. In late 89, +a brand-new Legion of Doom board, "Phoenix Project," +had gone up in Austin, Texas. Phoenix Project was sysoped +by no less a man than the Mentor himself, ably assisted by +University of Texas student and hardened Doomster "Erik Bloodaxe." + +As we have seen from his Phrack manifesto, the Mentor was a hacker +zealot who regarded computer intrusion as something close to a moral duty. +Phoenix Project was an ambitious effort, intended to revive the digital +underground to what Mentor considered the full flower of the early 80s. +The Phoenix board would also boldly bring elite hackers face-to-face +with the telco "opposition." On "Phoenix," America's cleverest hackers +would supposedly shame the telco squareheads out of their stick-in-the-mud +attitudes, and perhaps convince them that the Legion of Doom elite were really +an all-right crew. The premiere of "Phoenix Project" was heavily trumpeted +by Phrack,and "Phoenix Project" carried a complete run of Phrack issues, +including the E911 Document as Phrack had published it. + +Phoenix Project was only one of many--possibly hundreds--of nodes and boards +all over America that were in guilty possession of the E911 Document. +But Phoenix was an outright, unashamed Legion of Doom board. +Under Mentor's guidance, it was flaunting itself in the face +of telco security personnel. Worse yet, it was actively trying +to WIN THEM OVER as sympathizers for the digital underground elite. +"Phoenix" had no cards or codes on it. Its hacker elite considered +Phoenix at least technically legal. But Phoenix was a corrupting influence, +where hacker anarchy was eating away like digital acid at the underbelly +of corporate propriety. + +The Chicago Computer Fraud and Abuse Task Force now prepared +to descend upon Austin, Texas. + +Oddly, not one but TWO trails of the Task Force's investigation led +toward Austin. The city of Austin, like Atlanta, had made itself +a bulwark of the Sunbelt's Information Age, with a strong university +research presence, and a number of cutting-edge electronics companies, +including Motorola, Dell, CompuAdd, IBM, Sematech and MCC. + +Where computing machinery went, hackers generally followed. +Austin boasted not only "Phoenix Project," currently LoD's +most flagrant underground board, but a number of UNIX nodes. + +One of these nodes was "Elephant," run by a UNIX consultant +named Robert Izenberg. Izenberg, in search of a relaxed Southern +lifestyle and a lowered cost-of-living, had recently migrated +to Austin from New Jersey. In New Jersey, Izenberg had worked +for an independent contracting company, programming UNIX code for +AT&T itself. "Terminus" had been a frequent user on Izenberg's +privately owned Elephant node. + +Having interviewed Terminus and examined the records on Netsys, +the Chicago Task Force were now convinced that they had discovered +an underground gang of UNIX software pirates, who were demonstrably +guilty of interstate trafficking in illicitly copied AT&T source code. +Izenberg was swept into the dragnet around Terminus, the self-proclaimed +ultimate UNIX hacker. + +Izenberg, in Austin, had settled down into a UNIX job +with a Texan branch of IBM. Izenberg was no longer +working as a contractor for AT&T, but he had friends +in New Jersey, and he still logged on to AT&T UNIX +computers back in New Jersey, more or less whenever +it pleased him. Izenberg's activities appeared highly +suspicious to the Task Force. Izenberg might well be +breaking into AT&T computers, swiping AT&T software, +and passing it to Terminus and other possible confederates, +through the UNIX node network. And this data was worth, +not merely $79,499, but hundreds of thousands of dollars! + +On February 21, 1990, Robert Izenberg arrived home +from work at IBM to find that all the computers +had mysteriously vanished from his Austin apartment. +Naturally he assumed that he had been robbed. +His "Elephant" node, his other machines, his notebooks, +his disks, his tapes, all gone! However, nothing much +else seemed disturbed--the place had not been ransacked. +The puzzle becaming much stranger some five minutes later. +Austin U. S. Secret Service Agent Al Soliz, accompanied by +University of Texas campus-security officer Larry Coutorie +and the ubiquitous Tim Foley, made their appearance at Izenberg's door. +They were in plain clothes: slacks, polo shirts. They came in, +and Tim Foley accused Izenberg of belonging to the Legion of Doom. + +Izenberg told them that he had never heard of the "Legion of Doom." +And what about a certain stolen E911 Document, that posed a direct +threat to the police emergency lines? Izenberg claimed that he'd +never heard of that, either. + +His interrogators found this difficult to believe. +Didn't he know Terminus? + +Who? + +They gave him Terminus's real name. Oh yes, said Izenberg. +He knew THAT guy all right--he was leading discussions +on the Internet about AT&T computers, especially the AT&T 3B2. + +AT&T had thrust this machine into the marketplace, +but, like many of AT&T's ambitious attempts to enter +the computing arena, the 3B2 project had something less +than a glittering success. Izenberg himself had been +a contractor for the division of AT&T that supported the 3B2. +The entire division had been shut down. + +Nowadays, the cheapest and quickest way to get help with this +fractious piece of machinery was to join one of Terminus's +discussion groups on the Internet, where friendly and knowledgeable +hackers would help you for free. Naturally the remarks within this +group were less than flattering about the Death Star. . .was +THAT the problem? + +Foley told Izenberg that Terminus had been acquiring hot software +through his, Izenberg's, machine. + +Izenberg shrugged this off. A good eight megabytes of data flowed +through his UUCP site every day. UUCP nodes spewed data like fire hoses. +Elephant had been directly linked to Netsys--not surprising, since Terminus +was a 3B2 expert and Izenberg had been a 3B2 contractor. +Izenberg was also linked to "attctc" and the University of Texas. +Terminus was a well-known UNIX expert, and might have been up to +all manner of hijinks on Elephant. Nothing Izenberg could do about that. +That was physically impossible. Needle in a haystack. + +In a four-hour grilling, Foley urged Izenberg to come clean +and admit that he was in conspiracy with Terminus, +and a member of the Legion of Doom. + +Izenberg denied this. He was no weirdo teenage hacker-- +he was thirty-two years old, and didn't even have a "handle." +Izenberg was a former TV technician and electronics specialist +who had drifted into UNIX consulting as a full-grown adult. +Izenberg had never met Terminus, physically. He'd once bought +a cheap high-speed modem from him, though. + +Foley told him that this modem (a Telenet T2500 which ran at 19.2 kilobaud, +and which had just gone out Izenberg's door in Secret Service custody) +was likely hot property. Izenberg was taken aback to hear this; but then +again, most of Izenberg's equipment, like that of most freelance professionals +in the industry, was discounted, passed hand-to-hand through various kinds +of barter and gray-market. There was no proof that the modem was stolen, +and even if it were, Izenberg hardly saw how that gave them the right +to take every electronic item in his house. + +Still, if the United States Secret Service figured they needed +his computer for national security reasons--or whatever-- +then Izenberg would not kick. He figured he would somehow +make the sacrifice of his twenty thousand dollars' worth +of professional equipment, in the spirit of full cooperation +and good citizenship. + +Robert Izenberg was not arrested. Izenberg was not charged with any crime. +His UUCP node--full of some 140 megabytes of the files, mail, and data +of himself and his dozen or so entirely innocent users--went out the door +as "evidence." Along with the disks and tapes, Izenberg had lost about +800 megabytes of data. + +Six months would pass before Izenberg decided to phone the Secret Service +and ask how the case was going. That was the first time that Robert Izenberg +would ever hear the name of William Cook. As of January 1992, a full +two years after the seizure, Izenberg, still not charged with any crime, +would be struggling through the morass of the courts, in hope of recovering +his thousands of dollars' worth of seized equipment. + +In the meantime, the Izenberg case received absolutely no press coverage. +The Secret Service had walked into an Austin home, removed a UNIX bulletin- +board system, and met with no operational difficulties whatsoever. + +Except that word of a crackdown had percolated through the Legion of Doom. +"The Mentor" voluntarily shut down "The Phoenix Project." It seemed a pity, +especially as telco security employees had, in fact, shown up on Phoenix, +just as he had hoped--along with the usual motley crowd of LoD heavies, +hangers-on, phreaks, hackers and wannabes. There was "Sandy" Sandquist from +US SPRINT security, and some guy named Henry Kluepfel, from Bellcore itself! +Kluepfel had been trading friendly banter with hackers on Phoenix since +January 30th (two weeks after the Martin Luther King Day Crash). +The presence of such a stellar telco official seemed quite the coup +for Phoenix Project. + +Still, Mentor could judge the climate. Atlanta in ruins, +Phrack in deep trouble, something weird going on with UNIX nodes-- +discretion was advisable. Phoenix Project went off-line. + +Kluepfel, of course, had been monitoring this LoD bulletin +board for his own purposes--and those of the Chicago unit. +As far back as June 1987, Kluepfel had logged on to a Texas +underground board called "Phreak Klass 2600." There he'd +discovered an Chicago youngster named "Shadowhawk," +strutting and boasting about rifling AT&T computer files, +and bragging of his ambitions to riddle AT&T's Bellcore +computers with trojan horse programs. Kluepfel had passed +the news to Cook in Chicago, Shadowhawk's computers +had gone out the door in Secret Service custody, +and Shadowhawk himself had gone to jail. + +Now it was Phoenix Project's turn. Phoenix Project postured +about "legality" and "merely intellectual interest," but it reeked +of the underground. It had Phrack on it. It had the E911 Document. +It had a lot of dicey talk about breaking into systems, including some +bold and reckless stuff about a supposed "decryption service" that Mentor +and friends were planning to run, to help crack encrypted passwords off +of hacked systems. + +Mentor was an adult. There was a bulletin board at his place of work, +as well. Kleupfel logged onto this board, too, and discovered it to be +called "Illuminati." It was run by some company called Steve Jackson Games. + +On March 1, 1990, the Austin crackdown went into high gear. + +On the morning of March 1--a Thursday--21-year-old University of Texas +student "Erik Bloodaxe," co-sysop of Phoenix Project and an avowed member +of the Legion of Doom, was wakened by a police revolver levelled at his head. + +Bloodaxe watched, jittery, as Secret Service agents +appropriated his 300 baud terminal and, rifling his files, +discovered his treasured source-code for Robert Morris's +notorious Internet Worm. But Bloodaxe, a wily operator, +had suspected that something of the like might be coming. +All his best equipment had been hidden away elsewhere. +The raiders took everything electronic, however, +including his telephone. They were stymied by his +hefty arcade-style Pac-Man game, and left it in place, +as it was simply too heavy to move. + +Bloodaxe was not arrested. He was not charged with any crime. +A good two years later, the police still had what they had +taken from him, however. + +The Mentor was less wary. The dawn raid rousted him and his wife +from bed in their underwear, and six Secret Service agents, +accompanied by an Austin policeman and Henry Kluepfel himself, +made a rich haul. Off went the works, into the agents' white +Chevrolet minivan: an IBM PC-AT clone with 4 meg of RAM and +a 120-meg hard disk; a Hewlett-Packard LaserJet II printer; +a completely legitimate and highly expensive SCO-Xenix 286 +operating system; Pagemaker disks and documentation; +and the Microsoft Word word-processing program. Mentor's wife +had her incomplete academic thesis stored on the hard-disk; +that went, too, and so did the couple's telephone. As of two years later, +all this property remained in police custody. + +Mentor remained under guard in his apartment as agents prepared +to raid Steve Jackson Games. The fact that this was a business +headquarters and not a private residence did not deter the agents. +It was still very early; no one was at work yet. The agents prepared +to break down the door, but Mentor, eavesdropping on the Secret Service +walkie-talkie traffic, begged them not to do it, and offered his key +to the building. + +The exact details of the next events are unclear. The agents +would not let anyone else into the building. Their search warrant, +when produced, was unsigned. Apparently they breakfasted from the local +"Whataburger," as the litter from hamburgers was later found inside. +They also extensively sampled a bag of jellybeans kept by an SJG employee. +Someone tore a "Dukakis for President" sticker from the wall. + +SJG employees, diligently showing up for the day's work, were met +at the door and briefly questioned by U.S. Secret Service agents. +The employees watched in astonishment as agents wielding crowbars +and screwdrivers emerged with captive machines. They attacked +outdoor storage units with boltcutters. The agents wore +blue nylon windbreakers with "SECRET SERVICE" stencilled +across the back, with running-shoes and jeans. + +Jackson's company lost three computers, several hard-disks, +hundred of floppy disks, two monitors, three modems, +a laser printer, various powercords, cables, and adapters +(and, oddly, a small bag of screws, bolts and nuts). +The seizure of Illuminati BBS deprived SJG of all the programs, +text files, and private e-mail on the board. The loss of two other +SJG computers was a severe blow as well, since it caused the loss +of electronically stored contracts, financial projections, +address directories, mailing lists, personnel files, +business correspondence, and, not least, the drafts +of forthcoming games and gaming books. + +No one at Steve Jackson Games was arrested. No one was accused +of any crime. No charges were filed. Everything appropriated +was officially kept as "evidence" of crimes never specified. + +After the Phrack show-trial, the Steve Jackson Games scandal +was the most bizarre and aggravating incident of the Hacker +Crackdown of 1990. This raid by the Chicago Task Force +on a science-fiction gaming publisher was to rouse a +swarming host of civil liberties issues, and gave rise +to an enduring controversy that was still re-complicating itself, +and growing in the scope of its implications, a full two years later. + +The pursuit of the E911 Document stopped with the Steve Jackson Games raid. +As we have seen, there were hundreds, perhaps thousands of computer users +in America with the E911 Document in their possession. Theoretically, +Chicago had a perfect legal right to raid any of these people, +and could have legally seized the machines of anybody who subscribed to Phrack. +However, there was no copy of the E911 Document on Jackson's Illuminati board. +And there the Chicago raiders stopped dead; they have not raided anyone since. + +It might be assumed that Rich Andrews and Charlie Boykin, who had brought +the E911 Document to the attention of telco security, might be spared +any official suspicion. But as we have seen, the willingness to +"cooperate fully" offers little, if any, assurance against federal +anti-hacker prosecution. + +Richard Andrews found himself in deep trouble, thanks to the E911 Document. +Andrews lived in Illinois, the native stomping grounds of the Chicago +Task Force. On February 3 and 6, both his home and his place of work +were raided by USSS. His machines went out the door, too, and he was +grilled at length (though not arrested). Andrews proved to be in +purportedly guilty possession of: UNIX SVR 3.2; UNIX SVR 3.1; UUCP; +PMON; WWB; IWB; DWB; NROFF; KORN SHELL '88; C++; and QUEST, +among other items. Andrews had received this proprietary code-- +which AT&T officially valued at well over $250,000--through the +UNIX network, much of it supplied to him as a personal favor by Terminus. +Perhaps worse yet, Andrews admitted to returning the favor, by passing +Terminus a copy of AT&T proprietary STARLAN source code. + +Even Charles Boykin, himself an AT&T employee, entered some very hot water. +By 1990, he'd almost forgotten about the E911 problem he'd reported in +September 88; in fact, since that date, he'd passed two more security alerts +to Jerry Dalton, concerning matters that Boykin considered far worse than +the E911 Document. + +But by 1990, year of the crackdown, AT&T Corporate Information Security +was fed up with "Killer." This machine offered no direct income to AT&T, +and was providing aid and comfort to a cloud of suspicious yokels +from outside the company, some of them actively malicious toward AT&T, +its property, and its corporate interests. Whatever goodwill and publicity +had been won among Killer's 1,500 devoted users was considered no longer +worth the security risk. On February 20, 1990, Jerry Dalton arrived in +Dallas and simply unplugged the phone jacks, to the puzzled alarm +of Killer's many Texan users. Killer went permanently off-line, +with the loss of vast archives of programs and huge quantities +of electronic mail; it was never restored to service. AT&T showed +no particular regard for the "property" of these 1,500 people. +Whatever "property" the users had been storing on AT&T's computer +simply vanished completely. + +Boykin, who had himself reported the E911 problem, +now found himself under a cloud of suspicion. In a weird +private-security replay of the Secret Service seizures, +Boykin's own home was visited by AT&T Security and his +own machines were carried out the door. + +However, there were marked special features in the Boykin case. +Boykin's disks and his personal computers were swiftly examined +by his corporate employers and returned politely in just two days-- +(unlike Secret Service seizures, which commonly take months or years). +Boykin was not charged with any crime or wrongdoing, and he kept his job +with AT&T (though he did retire from AT&T in September 1991, +at the age of 52). + +It's interesting to note that the US Secret Service somehow failed +to seize Boykin's "Killer" node and carry AT&T's own computer out the door. +Nor did they raid Boykin's home. They seemed perfectly willing to take the +word of AT&T Security that AT&T's employee, and AT&T's "Killer" node, +were free of hacker contraband and on the up-and-up. + +It's digital water-under-the-bridge at this point, as Killer's +3,200 megabytes of Texan electronic community were erased in 1990, +and "Killer" itself was shipped out of the state. + +But the experiences of Andrews and Boykin, and the users of their systems, +remained side issues. They did not begin to assume the social, political, +and legal importance that gathered, slowly but inexorably, around the issue +of the raid on Steve Jackson Games. + +# + +We must now turn our attention to Steve Jackson Games itself, +and explain what SJG was, what it really did, and how it had +managed to attract this particularly odd and virulent kind of trouble. +The reader may recall that this is not the first but the second time +that the company has appeared in this narrative; a Steve Jackson game +called GURPS was a favorite pastime of Atlanta hacker Urvile, +and Urvile's science-fictional gaming notes had been mixed up +promiscuously with notes about his actual computer intrusions. + +First, Steve Jackson Games, Inc., was NOT a publisher of "computer games." +SJG published "simulation games," parlor games that were played on paper, +with pencils, and dice, and printed guidebooks full of rules and +statistics tables. There were no computers involved in the games themselves. +When you bought a Steve Jackson Game, you did not receive any software disks. +What you got was a plastic bag with some cardboard game tokens, +maybe a few maps or a deck of cards. Most of their products were books. + +However, computers WERE deeply involved in the Steve Jackson Games business. +Like almost all modern publishers, Steve Jackson and his fifteen employees +used computers to write text, to keep accounts, and to run the business +generally. They also used a computer to run their official bulletin board +system for Steve Jackson Games, a board called Illuminati. On Illuminati, +simulation gamers who happened to own computers and modems could associate, +trade mail, debate the theory and practice of gaming, and keep up with the +company's news and its product announcements. + +Illuminati was a modestly popular board, run on a small computer +with limited storage, only one phone-line, and no ties to large-scale +computer networks. It did, however, have hundreds of users, +many of them dedicated gamers willing to call from out-of-state. + +Illuminati was NOT an "underground" board. It did not feature hints +on computer intrusion, or "anarchy files," or illicitly posted +credit card numbers, or long-distance access codes. +Some of Illuminati's users, however, were members of the Legion of Doom. +And so was one of Steve Jackson's senior employees--the Mentor. +The Mentor wrote for Phrack, and also ran an underground board, +Phoenix Project--but the Mentor was not a computer professional. +The Mentor was the managing editor of Steve Jackson Games and +a professional game designer by trade. These LoD members did not +use Illuminati to help their HACKING activities. They used it to +help their GAME-PLAYING activities--and they were even more dedicated +to simulation gaming than they were to hacking. + +"Illuminati" got its name from a card-game that Steve Jackson himself, +the company's founder and sole owner, had invented. This multi-player +card-game was one of Mr Jackson's best-known, most successful, +most technically innovative products. "Illuminati" was a game +of paranoiac conspiracy in which various antisocial cults warred +covertly to dominate the world. "Illuminati" was hilarious, +and great fun to play, involving flying saucers, the CIA, the KGB, +the phone companies, the Ku Klux Klan, the South American Nazis, +the cocaine cartels, the Boy Scouts, and dozens of other splinter groups +from the twisted depths of Mr. Jackson's professionally fervid imagination. +For the uninitiated, any public discussion of the "Illuminati" card-game +sounded, by turns, utterly menacing or completely insane. + +And then there was SJG's "Car Wars," in which souped-up armored hot-rods +with rocket-launchers and heavy machine-guns did battle on the American +highways of the future. The lively Car Wars discussion on the Illuminati +board featured many meticulous, painstaking discussions of the effects +of grenades, land-mines, flamethrowers and napalm. It sounded like +hacker anarchy files run amuck. + +Mr Jackson and his co-workers earned their daily bread by supplying people +with make-believe adventures and weird ideas. The more far-out, the better. + +Simulation gaming is an unusual pastime, but gamers have not +generally had to beg the permission of the Secret Service to exist. +Wargames and role-playing adventures are an old and honored pastime, +much favored by professional military strategists. Once little-known, +these games are now played by hundreds of thousands of enthusiasts +throughout North America, Europe and Japan. Gaming-books, once restricted +to hobby outlets, now commonly appear in chain-stores like B. Dalton's +and Waldenbooks, and sell vigorously. + +Steve Jackson Games, Inc., of Austin, Texas, was a games company +of the middle rank. In 1989, SJG grossed about a million dollars. +Jackson himself had a good reputation in his industry as a talented +and innovative designer of rather unconventional games, but his company +was something less than a titan of the field--certainly not like the +multimillion-dollar TSR Inc., or Britain's gigantic "Games Workshop." +SJG's Austin headquarters was a modest two-story brick office-suite, +cluttered with phones, photocopiers, fax machines and computers. +It bustled with semi-organized activity and was littered with +glossy promotional brochures and dog-eared science-fiction novels. +Attached to the offices was a large tin-roofed warehouse piled twenty feet +high with cardboard boxes of games and books. Despite the weird imaginings +that went on within it, the SJG headquarters was quite a quotidian, +everyday sort of place. It looked like what it was: a publishers' digs. + +Both "Car Wars" and "Illuminati" were well-known, popular games. +But the mainstay of the Jackson organization was their Generic Universal +Role-Playing System, "G.U.R.P.S." The GURPS system was considered solid +and well-designed, an asset for players. But perhaps the most popular +feature of the GURPS system was that it allowed gaming-masters to design +scenarios that closely resembled well-known books, movies, and other works +of fantasy. Jackson had licensed and adapted works from many science fiction +and fantasy authors. There was GURPS Conan, GURPS Riverworld, +GURPS Horseclans, GURPS Witch World, names eminently familiar +to science-fiction readers. And there was GURPS Special Ops, +from the world of espionage fantasy and unconventional warfare. + +And then there was GURPS Cyberpunk. + +"Cyberpunk" was a term given to certain science fiction writers +who had entered the genre in the 1980s. "Cyberpunk," as the label implies, +had two general distinguishing features. First, its writers had a compelling +interest in information technology, an interest closely akin +to science fiction's earlier fascination with space travel. +And second, these writers were "punks," with all the +distinguishing features that that implies: Bohemian artiness, +youth run wild, an air of deliberate rebellion, funny clothes and hair, +odd politics, a fondness for abrasive rock and roll; in a word, trouble. + +The "cyberpunk" SF writers were a small group of mostly college-educated +white middle-class litterateurs, scattered through the US and Canada. +Only one, Rudy Rucker, a professor of computer science in Silicon Valley, +could rank with even the humblest computer hacker. But, except for +Professor Rucker, the "cyberpunk" authors were not programmers +or hardware experts; they considered themselves artists +(as, indeed, did Professor Rucker). However, these writers +all owned computers, and took an intense and public interest +in the social ramifications of the information industry. + +The cyberpunks had a strong following among the global generation +that had grown up in a world of computers, multinational networks, +and cable television. Their outlook was considered somewhat morbid, +cynical, and dark, but then again, so was the outlook of their +generational peers. As that generation matured and increased +in strength and influence, so did the cyberpunks. +As science-fiction writers went, they were doing +fairly well for themselves. By the late 1980s, +their work had attracted attention from gaming companies, +including Steve Jackson Games, which was planning a cyberpunk +simulation for the flourishing GURPS gaming-system. + +The time seemed ripe for such a product, which had already been proven +in the marketplace. The first games- company out of the gate, +with a product boldly called "Cyberpunk" in defiance of possible +infringement-of-copyright suits, had been an upstart group called +R. Talsorian. Talsorian's Cyberpunk was a fairly decent game, +but the mechanics of the simulation system left a lot to be desired. +Commercially, however, the game did very well. + +The next cyberpunk game had been the even more successful Shadowrun +by FASA Corporation. The mechanics of this game were fine, but the +scenario was rendered moronic by sappy fantasy elements like elves, +trolls, wizards, and dragons--all highly ideologically-incorrect, +according to the hard-edged, high-tech standards of cyberpunk science fiction. + +Other game designers were champing at the bit. Prominent among them +was the Mentor, a gentleman who, like most of his friends in the +Legion of Doom, was quite the cyberpunk devotee. Mentor reasoned +that the time had come for a REAL cyberpunk gaming-book--one that the +princes of computer-mischief in the Legion of Doom could play without +laughing themselves sick. This book, GURPS Cyberpunk, would reek +of culturally on-line authenticity. + +Mentor was particularly well-qualified for this task. +Naturally, he knew far more about computer-intrusion +and digital skullduggery than any previously published +cyberpunk author. Not only that, but he was good at his work. +A vivid imagination, combined with an instinctive feeling +for the working of systems and, especially, the loopholes +within them, are excellent qualities for a professional game designer. + +By March 1st, GURPS Cyberpunk was almost complete, ready to print and ship. +Steve Jackson expected vigorous sales for this item, which, he hoped, +would keep the company financially afloat for several months. +GURPS Cyberpunk, like the other GURPS "modules," was not a "game" +like a Monopoly set, but a BOOK: a bound paperback book the size +of a glossy magazine, with a slick color cover, and pages full of text, +illustrations, tables and footnotes. It was advertised as a game, +and was used as an aid to game-playing, but it was a book, +with an ISBN number, published in Texas, copyrighted, +and sold in bookstores. + +And now, that book, stored on a computer, had gone out the door +in the custody of the Secret Service. + +The day after the raid, Steve Jackson visited the local Secret Service +headquarters with a lawyer in tow. There he confronted Tim Foley +(still in Austin at that time) and demanded his book back. But there +was trouble. GURPS Cyberpunk, alleged a Secret Service agent to astonished +businessman Steve Jackson, was "a manual for computer crime." + +"It's science fiction," Jackson said. + +"No, this is real." + +This statement was repeated several times, by several agents. +Jackson's ominously accurate game had passed from pure, +obscure, small-scale fantasy into the impure, highly publicized, +large-scale fantasy of the Hacker Crackdown. + +No mention was made of the real reason for the search. +According to their search warrant, the raiders had expected +to find the E911 Document stored on Jackson's bulletin board system. +But that warrant was sealed; a procedure that most law enforcement agencies +will use only when lives are demonstrably in danger. The raiders' +true motives were not discovered until the Jackson search-warrant +was unsealed by his lawyers, many months later. The Secret Service, +and the Chicago Computer Fraud and Abuse Task Force, +said absolutely nothing to Steve Jackson about any threat +to the police 911 System. They said nothing about the Atlanta Three, +nothing about Phrack or Knight Lightning, nothing about Terminus. + +Jackson was left to believe that his computers had been seized because +he intended to publish a science fiction book that law enforcement +considered too dangerous to see print. + +This misconception was repeated again and again, for months, +to an ever-widening public audience. It was not the truth of the case; +but as months passed, and this misconception was publicly printed again +and again, it became one of the few publicly known "facts" about +the mysterious Hacker Crackdown. The Secret Service had seized a computer +to stop the publication of a cyberpunk science fiction book. + +The second section of this book, "The Digital Underground," +is almost finished now. We have become acquainted with all +the major figures of this case who actually belong to the +underground milieu of computer intrusion. We have some idea +of their history, their motives, their general modus operandi. +We now know, I hope, who they are, where they came from, +and more or less what they want. In the next section of this book, +"Law and Order," we will leave this milieu and directly enter the +world of America's computer-crime police. + +At this point, however, I have another figure to introduce: myself. + +My name is Bruce Sterling. I live in Austin, Texas, where I am +a science fiction writer by trade: specifically, a CYBERPUNK +science fiction writer. + +Like my "cyberpunk" colleagues in the U.S. and Canada, +I've never been entirely happy with this literary label-- +especially after it became a synonym for computer criminal. +But I did once edit a book of stories by my colleagues, +called Mirrorshades: the Cyberpunk Anthology, and I've +long been a writer of literary-critical cyberpunk manifestos. +I am not a "hacker" of any description, though I do have readers +in the digital underground. + +When the Steve Jackson Games seizure occurred, I naturally took +an intense interest. If "cyberpunk" books were being banned +by federal police in my own home town, I reasonably wondered +whether I myself might be next. Would my computer be seized +by the Secret Service? At the time, I was in possession +of an aging Apple IIe without so much as a hard disk. +If I were to be raided as an author of computer-crime manuals, +the loss of my feeble word-processor would likely provoke more +snickers than sympathy. + +I'd known Steve Jackson for many years. We knew +one another as colleagues, for we frequented +the same local science-fiction conventions. +I'd played Jackson games, and recognized his cleverness; +but he certainly had never struck me as a potential mastermind +of computer crime. + +I also knew a little about computer bulletin-board systems. +In the mid-1980s I had taken an active role in an Austin board +called "SMOF-BBS," one of the first boards dedicated to science fiction. +I had a modem, and on occasion I'd logged on to Illuminati, +which always looked entertainly wacky, but certainly harmless enough. + +At the time of the Jackson seizure, I had no experience +whatsoever with underground boards. But I knew that no one +on Illuminati talked about breaking into systems illegally, +or about robbing phone companies. Illuminati didn't even +offer pirated computer games. Steve Jackson, like many creative artists, +was markedly touchy about theft of intellectual property. + +It seemed to me that Jackson was either seriously suspected +of some crime--in which case, he would be charged soon, +and would have his day in court--or else he was innocent, +in which case the Secret Service would quickly return his equipment, +and everyone would have a good laugh. I rather expected the good laugh. +The situation was not without its comic side. The raid, known +as the "Cyberpunk Bust" in the science fiction community, +was winning a great deal of free national publicity both +for Jackson himself and the "cyberpunk" science fiction +writers generally. + +Besides, science fiction people are used to being misinterpreted. +Science fiction is a colorful, disreputable, slipshod occupation, +full of unlikely oddballs, which, of course, is why we like it. +Weirdness can be an occupational hazard in our field. People who +wear Halloween costumes are sometimes mistaken for monsters. + +Once upon a time--back in 1939, in New York City-- +science fiction and the U.S. Secret Service collided in +a comic case of mistaken identity. This weird incident +involved a literary group quite famous in science fiction, +known as "the Futurians," whose membership included +such future genre greats as Isaac Asimov, Frederik Pohl, +and Damon Knight. The Futurians were every bit as +offbeat and wacky as any of their spiritual descendants, +including the cyberpunks, and were given to communal living, +spontaneous group renditions of light opera, and midnight fencing +exhibitions on the lawn. The Futurians didn't have bulletin +board systems, but they did have the technological equivalent +in 1939--mimeographs and a private printing press. These were +in steady use, producing a stream of science-fiction fan magazines, +literary manifestos, and weird articles, which were picked up +in ink-sticky bundles by a succession of strange, gangly, +spotty young men in fedoras and overcoats. + +The neighbors grew alarmed at the antics of the Futurians +and reported them to the Secret Service as suspected counterfeiters. +In the winter of 1939, a squad of USSS agents with drawn guns burst into +"Futurian House," prepared to confiscate the forged currency and illicit +printing presses. There they discovered a slumbering science fiction fan +named George Hahn, a guest of the Futurian commune who had just arrived +in New York. George Hahn managed to explain himself and his group, +and the Secret Service agents left the Futurians in peace henceforth. +(Alas, Hahn died in 1991, just before I had discovered this astonishing +historical parallel, and just before I could interview him for this book.) + +But the Jackson case did not come to a swift and comic end. +No quick answers came his way, or mine; no swift reassurances +that all was right in the digital world, that matters were well +in hand after all. Quite the opposite. In my alternate role +as a sometime pop-science journalist, I interviewed Jackson +and his staff for an article in a British magazine. +The strange details of the raid left me more concerned than ever. +Without its computers, the company had been financially +and operationally crippled. Half the SJG workforce, +a group of entirely innocent people, had been sorrowfully fired, +deprived of their livelihoods by the seizure. It began to dawn on me +that authors--American writers--might well have their computers seized, +under sealed warrants, without any criminal charge; and that, +as Steve Jackson had discovered, there was no immediate recourse for this. +This was no joke; this wasn't science fiction; this was real. + +I determined to put science fiction aside until I had discovered +what had happened and where this trouble had come from. +It was time to enter the purportedly real world of electronic +free expression and computer crime. Hence, this book. +Hence, the world of the telcos; and the world of the digital underground; +and next, the world of the police. + + + +PART THREE: LAW AND ORDER + + +Of the various anti-hacker activities of 1990, "Operation Sundevil" +had by far the highest public profile. The sweeping, nationwide +computer seizures of May 8, 1990 were unprecedented in scope and highly, +if rather selectively, publicized. + +Unlike the efforts of the Chicago Computer Fraud and Abuse Task Force, +"Operation Sundevil" was not intended to combat "hacking" in the sense +of computer intrusion or sophisticated raids on telco switching stations. +Nor did it have anything to do with hacker misdeeds with AT&T's software, +or with Southern Bell's proprietary documents. + +Instead, "Operation Sundevil" was a crackdown on those traditional scourges +of the digital underground: credit-card theft and telephone code abuse. +The ambitious activities out of Chicago, and the somewhat lesser-known +but vigorous anti-hacker actions of the New York State Police in 1990, +were never a part of "Operation Sundevil" per se, which was based in Arizona. + +Nevertheless, after the spectacular May 8 raids, the public, misled by +police secrecy, hacker panic, and a puzzled national press-corps, +conflated all aspects of the nationwide crackdown in 1990 under +the blanket term "Operation Sundevil." "Sundevil" is still the best-known +synonym for the crackdown of 1990. But the Arizona organizers of "Sundevil" +did not really deserve this reputation--any more, for instance, than all +hackers deserve a reputation as "hackers." + +There was some justice in this confused perception, though. +For one thing, the confusion was abetted by the Washington office +of the Secret Service, who responded to Freedom of Information Act +requests on "Operation Sundevil" by referring investigators +to the publicly known cases of Knight Lightning and the Atlanta Three. +And "Sundevil" was certainly the largest aspect of the Crackdown, +the most deliberate and the best-organized. As a crackdown on electronic +fraud, "Sundevil" lacked the frantic pace of the war on the Legion of Doom; +on the contrary, Sundevil's targets were picked out with cool deliberation +over an elaborate investigation lasting two full years. + +And once again the targets were bulletin board systems. + +Boards can be powerful aids to organized fraud. Underground boards carry +lively, extensive, detailed, and often quite flagrant "discussions" of +lawbreaking techniques and lawbreaking activities. "Discussing" crime +in the abstract, or "discussing" the particulars of criminal cases, +is not illegal--but there are stern state and federal laws against +coldbloodedly conspiring in groups in order to commit crimes. + +In the eyes of police, people who actively conspire to break the law +are not regarded as "clubs," "debating salons," "users' groups," or +"free speech advocates." Rather, such people tend to find themselves +formally indicted by prosecutors as "gangs," "racketeers," "corrupt +organizations" and "organized crime figures." + +What's more, the illicit data contained on outlaw boards goes well beyond +mere acts of speech and/or possible criminal conspiracy. As we have seen, +it was common practice in the digital underground to post purloined telephone +codes on boards, for any phreak or hacker who cared to abuse them. Is posting +digital booty of this sort supposed to be protected by the First Amendment? +Hardly--though the issue, like most issues in cyberspace, is not entirely +resolved. Some theorists argue that to merely RECITE a number publicly +is not illegal--only its USE is illegal. But anti-hacker police point out +that magazines and newspapers (more traditional forms of free expression) +never publish stolen telephone codes (even though this might well +raise their circulation). + +Stolen credit card numbers, being riskier and more valuable, +were less often publicly posted on boards--but there is no question +that some underground boards carried "carding" traffic, +generally exchanged through private mail. + +Underground boards also carried handy programs for "scanning" telephone +codes and raiding credit card companies, as well as the usual obnoxious +galaxy of pirated software, cracked passwords, blue-box schematics, +intrusion manuals, anarchy files, porn files, and so forth. + +But besides their nuisance potential for the spread of illicit knowledge, +bulletin boards have another vitally interesting aspect for the +professional investigator. Bulletin boards are cram-full of EVIDENCE. +All that busy trading of electronic mail, all those hacker boasts, +brags and struts, even the stolen codes and cards, can be neat, +electronic, real-time recordings of criminal activity. +As an investigator, when you seize a pirate board, you have +scored a coup as effective as tapping phones or intercepting mail. +However, you have not actually tapped a phone or intercepted a letter. +The rules of evidence regarding phone-taps and mail interceptions are old, +stern and well-understood by police, prosecutors and defense attorneys alike. +The rules of evidence regarding boards are new, waffling, and understood +by nobody at all. + +Sundevil was the largest crackdown on boards in world history. +On May 7, 8, and 9, 1990, about forty-two computer systems were seized. +Of those forty-two computers, about twenty-five actually were running boards. +(The vagueness of this estimate is attributable to the vagueness of +(a) what a "computer system" is, and (b) what it actually means to +"run a board" with one--or with two computers, or with three.) + +About twenty-five boards vanished into police custody in May 1990. +As we have seen, there are an estimated 30,000 boards in America today. +If we assume that one board in a hundred is up to no good with codes +and cards (which rather flatters the honesty of the board-using community), +then that would leave 2,975 outlaw boards untouched by Sundevil. +Sundevil seized about one tenth of one percent of all computer +bulletin boards in America. Seen objectively, this is something less +than a comprehensive assault. In 1990, Sundevil's organizers-- +the team at the Phoenix Secret Service office, and the Arizona +Attorney General's office-- had a list of at least THREE HUNDRED +boards that they considered fully deserving of search and seizure warrants. +The twenty-five boards actually seized were merely among the most obvious +and egregious of this much larger list of candidates. All these boards +had been examined beforehand--either by informants, who had passed printouts +to the Secret Service, or by Secret Service agents themselves, who not only +come equipped with modems but know how to use them. + +There were a number of motives for Sundevil. First, it offered +a chance to get ahead of the curve on wire-fraud crimes. +Tracking back credit-card ripoffs to their perpetrators +can be appallingly difficult. If these miscreants +have any kind of electronic sophistication, they can snarl +their tracks through the phone network into a mind-boggling, +untraceable mess, while still managing to "reach out and rob someone." +Boards, however, full of brags and boasts, codes and cards, +offer evidence in the handy congealed form. + +Seizures themselves--the mere physical removal of machines-- +tends to take the pressure off. During Sundevil, a large number +of code kids, warez d00dz, and credit card thieves would be deprived +of those boards--their means of community and conspiracy--in one swift blow. +As for the sysops themselves (commonly among the boldest offenders) +they would be directly stripped of their computer equipment, +and rendered digitally mute and blind. + +And this aspect of Sundevil was carried out with great success. +Sundevil seems to have been a complete tactical surprise-- +unlike the fragmentary and continuing seizures of the war on the +Legion of Doom, Sundevil was precisely timed and utterly overwhelming. +At least forty "computers" were seized during May 7, 8 and 9, 1990, +in Cincinnati, Detroit, Los Angeles, Miami, Newark, Phoenix, Tucson, +Richmond, San Diego, San Jose, Pittsburgh and San Francisco. +Some cities saw multiple raids, such as the five separate raids +in the New York City environs. Plano, Texas (essentially a suburb of +the Dallas/Fort Worth metroplex, and a hub of the telecommunications industry) +saw four computer seizures. Chicago, ever in the forefront, saw its own +local Sundevil raid, briskly carried out by Secret Service agents +Timothy Foley and Barbara Golden. + +Many of these raids occurred, not in the cities proper, +but in associated white-middle class suburbs--places like +Mount Lebanon, Pennsylvania and Clark Lake, Michigan. +There were a few raids on offices; most took place in people's homes, +the classic hacker basements and bedrooms. + +The Sundevil raids were searches and seizures, not a group of mass arrests. +There were only four arrests during Sundevil. "Tony the Trashman," +a longtime teenage bete noire of the Arizona Racketeering unit, +was arrested in Tucson on May 9. "Dr. Ripco," sysop of an outlaw board +with the misfortune to exist in Chicago itself, was also arrested-- +on illegal weapons charges. Local units also arrested a 19-year-old +female phone phreak named "Electra" in Pennsylvania, and a male juvenile +in California. Federal agents however were not seeking arrests, but computers. + +Hackers are generally not indicted (if at all) until the evidence +in their seized computers is evaluated--a process that can take weeks, +months--even years. When hackers are arrested on the spot, it's generally +an arrest for other reasons. Drugs and/or illegal weapons show up in a good +third of anti-hacker computer seizures (though not during Sundevil). + +That scofflaw teenage hackers (or their parents) should have marijuana +in their homes is probably not a shocking revelation, but the surprisingly +common presence of illegal firearms in hacker dens is a bit disquieting. +A Personal Computer can be a great equalizer for the techno-cowboy-- +much like that more traditional American "Great Equalizer," +the Personal Sixgun. Maybe it's not all that surprising +that some guy obsessed with power through illicit technology +would also have a few illicit high-velocity-impact devices around. +An element of the digital underground particularly dotes on those +"anarchy philes," and this element tends to shade into the crackpot milieu +of survivalists, gun-nuts, anarcho-leftists and the ultra-libertarian +right-wing. + +This is not to say that hacker raids to date have uncovered any +major crack-dens or illegal arsenals; but Secret Service agents +do not regard "hackers" as "just kids." They regard hackers as +unpredictable people, bright and slippery. It doesn't help matters +that the hacker himself has been "hiding behind his keyboard" +all this time. Commonly, police have no idea what he looks like. +This makes him an unknown quantity, someone best treated with +proper caution. + +To date, no hacker has come out shooting, though they do sometimes brag on +boards that they will do just that. Threats of this sort are taken seriously. +Secret Service hacker raids tend to be swift, comprehensive, well-manned +(even over-manned); and agents generally burst through every door +in the home at once, sometimes with drawn guns. Any potential resistance +is swiftly quelled. Hacker raids are usually raids on people's homes. +It can be a very dangerous business to raid an American home; +people can panic when strangers invade their sanctum. Statistically speaking, +the most dangerous thing a policeman can do is to enter someone's home. +(The second most dangerous thing is to stop a car in traffic.) +People have guns in their homes. More cops are hurt in homes +than are ever hurt in biker bars or massage parlors. + +But in any case, no one was hurt during Sundevil, +or indeed during any part of the Hacker Crackdown. + +Nor were there any allegations of any physical mistreatment of a suspect. +Guns were pointed, interrogations were sharp and prolonged; but no one +in 1990 claimed any act of brutality by any crackdown raider. + +In addition to the forty or so computers, Sundevil reaped floppy disks +in particularly great abundance--an estimated 23,000 of them, which +naturally included every manner of illegitimate data: pirated games, +stolen codes, hot credit card numbers, the complete text and software +of entire pirate bulletin-boards. These floppy disks, which remain +in police custody today, offer a gigantic, almost embarrassingly +rich source of possible criminal indictments. These 23,000 floppy disks +also include a thus-far unknown quantity of legitimate computer games, +legitimate software, purportedly "private" mail from boards, +business records, and personal correspondence of all kinds. + +Standard computer-crime search warrants lay great emphasis on seizing +written documents as well as computers--specifically including photocopies, +computer printouts, telephone bills, address books, logs, notes, +memoranda and correspondence. In practice, this has meant that diaries, +gaming magazines, software documentation, nonfiction books on hacking +and computer security, sometimes even science fiction novels, have all +vanished out the door in police custody. A wide variety of electronic items +have been known to vanish as well, including telephones, televisions, answering +machines, Sony Walkmans, desktop printers, compact disks, and audiotapes. + +No fewer than 150 members of the Secret Service were sent into +the field during Sundevil. They were commonly accompanied by +squads of local and/or state police. Most of these officers-- +especially the locals--had never been on an anti-hacker raid before. +(This was one good reason, in fact, why so many of them were invited along +in the first place.) Also, the presence of a uniformed police officer +assures the raidees that the people entering their homes are, in fact, police. +Secret Service agents wear plain clothes. So do the telco security experts +who commonly accompany the Secret Service on raids (and who make no particular +effort to identify themselves as mere employees of telephone companies). + +A typical hacker raid goes something like this. First, police storm in +rapidly, through every entrance, with overwhelming force, +in the assumption that this tactic will keep casualties to a minimum. +Second, possible suspects are immediately removed from the vicinity +of any and all computer systems, so that they will have no chance +to purge or destroy computer evidence. Suspects are herded into a room +without computers, commonly the living room, and kept under guard-- +not ARMED guard, for the guns are swiftly holstered, but under guard +nevertheless. They are presented with the search warrant and warned +that anything they say may be held against them. Commonly they have +a great deal to say, especially if they are unsuspecting parents. + +Somewhere in the house is the "hot spot"--a computer tied to a phone +line (possibly several computers and several phones). Commonly it's +a teenager's bedroom, but it can be anywhere in the house; +there may be several such rooms. This "hot spot" is put in charge +of a two-agent team, the "finder" and the "recorder." The "finder" +is computer-trained, commonly the case agent who has actually obtained +the search warrant from a judge. He or she understands what is being sought, +and actually carries out the seizures: unplugs machines, opens drawers, +desks, files, floppy-disk containers, etc. The "recorder" photographs +all the equipment, just as it stands--especially the tangle of +wired connections in the back, which can otherwise be a real nightmare +to restore. The recorder will also commonly photograph every room +in the house, lest some wily criminal claim that the police had robbed him +during the search. Some recorders carry videocams or tape recorders; +however, it's more common for the recorder to simply take written notes. +Objects are described and numbered as the finder seizes them, generally +on standard preprinted police inventory forms. + +Even Secret Service agents were not, and are not, expert computer users. +They have not made, and do not make, judgements on the fly about potential +threats posed by various forms of equipment. They may exercise discretion; +they may leave Dad his computer, for instance, but they don't HAVE to. +Standard computer-crime search warrants, which date back to the early 80s, +use a sweeping language that targets computers, most anything attached +to a computer, most anything used to operate a computer--most anything +that remotely resembles a computer--plus most any and all written documents +surrounding it. Computer-crime investigators have strongly urged agents +to seize the works. + +In this sense, Operation Sundevil appears to have been a complete success. +Boards went down all over America, and were shipped en masse to the computer +investigation lab of the Secret Service, in Washington DC, along with the +23,000 floppy disks and unknown quantities of printed material. + +But the seizure of twenty-five boards, and the multi-megabyte mountains +of possibly useful evidence contained in these boards (and in their owners' +other computers, also out the door), were far from the only motives for +Operation Sundevil. An unprecedented action of great ambition and size, +Sundevil's motives can only be described as political. It was a +public-relations effort, meant to pass certain messages, meant to make +certain situations clear: both in the mind of the general public, +and in the minds of various constituencies of the electronic community. + + First --and this motivation was vital--a "message" would be sent from +law enforcement to the digital underground. This very message was recited +in so many words by Garry M. Jenkins, the Assistant Director of the +US Secret Service, at the Sundevil press conference in Phoenix on +May 9, 1990, immediately after the raids. In brief, hackers were +mistaken in their foolish belief that they could hide behind the +"relative anonymity of their computer terminals." On the contrary, +they should fully understand that state and federal cops were +actively patrolling the beat in cyberspace--that they were +on the watch everywhere, even in those sleazy and secretive +dens of cybernetic vice, the underground boards. + +This is not an unusual message for police to publicly convey to crooks. +The message is a standard message; only the context is new. + +In this respect, the Sundevil raids were the digital equivalent +of the standard vice-squad crackdown on massage parlors, porno bookstores, +head-shops, or floating crap-games. There may be few or no arrests in a raid +of this sort; no convictions, no trials, no interrogations. In cases of this +sort, police may well walk out the door with many pounds of sleazy magazines, +X-rated videotapes, sex toys, gambling equipment, baggies of marijuana. . . . + +Of course, if something truly horrendous is discovered by the raiders, +there will be arrests and prosecutions. Far more likely, however, +there will simply be a brief but sharp disruption of the closed +and secretive world of the nogoodniks. There will be "street hassle." +"Heat." "Deterrence." And, of course, the immediate loss of the seized goods. +It is very unlikely that any of this seized material will ever be returned. +Whether charged or not, whether convicted or not, the perpetrators will +almost surely lack the nerve ever to ask for this stuff to be given back. + +Arrests and trials--putting people in jail--may involve all kinds of +formal legalities; but dealing with the justice system is far from the only +task of police. Police do not simply arrest people. They don't simply +put people in jail. That is not how the police perceive their jobs. +Police "protect and serve." Police "keep the peace," they "keep public order." +Like other forms of public relations, keeping public order is not an +exact science. Keeping public order is something of an art-form. + +If a group of tough-looking teenage hoodlums was loitering on a street-corner, +no one would be surprised to see a street-cop arrive and sternly order +them to "break it up." On the contrary, the surprise would come if one +of these ne'er-do-wells stepped briskly into a phone-booth, +called a civil rights lawyer, and instituted a civil suit +in defense of his Constitutional rights of free speech +and free assembly. But something much along this line +was one of the many anomolous outcomes of the Hacker Crackdown. + +Sundevil also carried useful "messages" for other constituents of +the electronic community. These messages may not have been read +aloud from the Phoenix podium in front of the press corps, +but there was little mistaking their meaning. There was a message +of reassurance for the primary victims of coding and carding: +the telcos, and the credit companies. Sundevil was greeted with joy +by the security officers of the electronic business community. +After years of high-tech harassment and spiralling revenue losses, +their complaints of rampant outlawry were being taken seriously by +law enforcement. No more head-scratching or dismissive shrugs; +no more feeble excuses about "lack of computer-trained officers" or +the low priority of "victimless" white-collar telecommunication crimes. + +Computer-crime experts have long believed that computer-related offenses +are drastically under-reported. They regard this as a major open scandal +of their field. Some victims are reluctant to come forth, because they +believe that police and prosecutors are not computer-literate, +and can and will do nothing. Others are embarrassed by +their vulnerabilities, and will take strong measures +to avoid any publicity; this is especially true of banks, +who fear a loss of investor confidence should an embezzlement-case +or wire-fraud surface. And some victims are so helplessly confused +by their own high technology that they never even realize that +a crime has occurred--even when they have been fleeced to the bone. + +The results of this situation can be dire. +Criminals escape apprehension and punishment. +The computer-crime units that do exist, can't get work. +The true scope of computer-crime: its size, its real nature, +the scope of its threats, and the legal remedies for it-- +all remain obscured. + +Another problem is very little publicized, but it is a cause +of genuine concern. Where there is persistent crime, +but no effective police protection, then vigilantism can result. +Telcos, banks, credit companies, the major corporations who +maintain extensive computer networks vulnerable to hacking +--these organizations are powerful, wealthy, and +politically influential. They are disinclined to be +pushed around by crooks (or by most anyone else, +for that matter). They often maintain well-organized +private security forces, commonly run by +experienced veterans of military and police units, +who have left public service for the greener pastures +of the private sector. For police, the corporate +security manager can be a powerful ally; but if this +gentleman finds no allies in the police, and the +pressure is on from his board-of-directors, +he may quietly take certain matters into his own hands. + +Nor is there any lack of disposable hired-help in the +corporate security business. Private security agencies-- +the `security business' generally--grew explosively in the 1980s. +Today there are spooky gumshoed armies of "security consultants," +"rent-a- cops," "private eyes," "outside experts"--every manner +of shady operator who retails in "results" and discretion. +Or course, many of these gentlemen and ladies may be paragons +of professional and moral rectitude. But as anyone +who has read a hard-boiled detective novel knows, +police tend to be less than fond of this sort +of private-sector competition. + +Companies in search of computer-security have even been +known to hire hackers. Police shudder at this prospect. + +Police treasure good relations with the business community. +Rarely will you see a policeman so indiscreet as to allege +publicly that some major employer in his state or city has succumbed +to paranoia and gone off the rails. Nevertheless, +police --and computer police in particular--are aware +of this possibility. Computer-crime police can and do +spend up to half of their business hours just doing +public relations: seminars, "dog and pony shows," +sometimes with parents' groups or computer users, +but generally with their core audience: the likely +victims of hacking crimes. These, of course, are telcos, +credit card companies and large computer-equipped corporations. +The police strongly urge these people, as good citizens, +to report offenses and press criminal charges; +they pass the message that there is someone in authority who cares, +understands, and, best of all, will take useful action +should a computer-crime occur. + +But reassuring talk is cheap. Sundevil offered action. + +The final message of Sundevil was intended for internal consumption +by law enforcement. Sundevil was offered as proof that the community +of American computer-crime police had come of age. Sundevil was +proof that enormous things like Sundevil itself could now be accomplished. +Sundevil was proof that the Secret Service and its local law-enforcement +allies could act like a well-oiled machine--(despite the hampering use +of those scrambled phones). It was also proof that the Arizona Organized +Crime and Racketeering Unit--the sparkplug of Sundevil--ranked with the best +in the world in ambition, organization, and sheer conceptual daring. + +And, as a final fillip, Sundevil was a message from the Secret Service +to their longtime rivals in the Federal Bureau of Investigation. +By Congressional fiat, both USSS and FBI formally share jurisdiction +over federal computer-crimebusting activities. Neither of these groups +has ever been remotely happy with this muddled situation. It seems to +suggest that Congress cannot make up its mind as to which of these groups +is better qualified. And there is scarcely a G-man or a Special Agent +anywhere without a very firm opinion on that topic. + +# + +For the neophyte, one of the most puzzling aspects of the crackdown +on hackers is why the United States Secret Service has anything at all +to do with this matter. + +The Secret Service is best known for its primary public role: +its agents protect the President of the United States. +They also guard the President's family, the Vice President and his family, +former Presidents, and Presidential candidates. They sometimes guard +foreign dignitaries who are visiting the United States, especially foreign +heads of state, and have been known to accompany American officials +on diplomatic missions overseas. + +Special Agents of the Secret Service don't wear uniforms, but the +Secret Service also has two uniformed police agencies. There's the +former White House Police (now known as the Secret Service Uniformed Division, +since they currently guard foreign embassies in Washington, as well as the +White House itself). And there's the uniformed Treasury Police Force. + +The Secret Service has been charged by Congress with a number +of little-known duties. They guard the precious metals in Treasury vaults. +They guard the most valuable historical documents of the United States: +originals of the Constitution, the Declaration of Independence, +Lincoln's Second Inaugural Address, an American-owned copy of +the Magna Carta, and so forth. Once they were assigned to guard +the Mona Lisa, on her American tour in the 1960s. + +The entire Secret Service is a division of the Treasury Department. +Secret Service Special Agents (there are about 1,900 of them) +are bodyguards for the President et al, but they all work for the Treasury. +And the Treasury (through its divisions of the U.S. Mint and the +Bureau of Engraving and Printing) prints the nation's money. + +As Treasury police, the Secret Service guards the nation's currency; +it is the only federal law enforcement agency with direct jurisdiction +over counterfeiting and forgery. It analyzes documents for authenticity, +and its fight against fake cash is still quite lively (especially since +the skilled counterfeiters of Medellin, Columbia have gotten into the act). +Government checks, bonds, and other obligations, which exist in untold +millions and are worth untold billions, are common targets for forgery, +which the Secret Service also battles. It even handles forgery +of postage stamps. + +But cash is fading in importance today as money has become electronic. +As necessity beckoned, the Secret Service moved from fighting the +counterfeiting of paper currency and the forging of checks, +to the protection of funds transferred by wire. + +From wire-fraud, it was a simple skip-and-jump to what is formally +known as "access device fraud." Congress granted the Secret Service +the authority to investigate "access device fraud" under Title 18 +of the United States Code (U.S.C. Section 1029). + +The term "access device" seems intuitively simple. It's some kind +of high-tech gizmo you use to get money with. It makes good sense +to put this sort of thing in the charge of counterfeiting and +wire-fraud experts. + +However, in Section 1029, the term "access device" is very +generously defined. An access device is: "any card, plate, +code, account number, or other means of account access +that can be used, alone or in conjunction with another access device, +to obtain money, goods, services, or any other thing of value, +or that can be used to initiate a transfer of funds." + +"Access device" can therefore be construed to include credit cards +themselves (a popular forgery item nowadays). It also includes credit card +account NUMBERS, those standards of the digital underground. The same goes +for telephone charge cards (an increasingly popular item with telcos, +who are tired of being robbed of pocket change by phone-booth thieves). +And also telephone access CODES, those OTHER standards of the digital +underground. (Stolen telephone codes may not "obtain money," but they +certainly do obtain valuable "services," which is specifically forbidden +by Section 1029.) + +We can now see that Section 1029 already pits the United States Secret Service +directly against the digital underground, without any mention at all of +the word "computer." + +Standard phreaking devices, like "blue boxes," used to steal phone service +from old-fashioned mechanical switches, are unquestionably "counterfeit +access devices." Thanks to Sec.1029, it is not only illegal to USE +counterfeit access devices, but it is even illegal to BUILD them. +"Producing," "designing" "duplicating" or "assembling" blue boxes +are all federal crimes today, and if you do this, the Secret Service +has been charged by Congress to come after you. + +Automatic Teller Machines, which replicated all over America during the 1980s, +are definitely "access devices," too, and an attempt to tamper with their +punch-in codes and plastic bank cards falls directly under Sec. 1029. + +Section 1029 is remarkably elastic. Suppose you find a computer password +in somebody's trash. That password might be a "code"--it's certainly a +"means of account access." Now suppose you log on to a computer +and copy some software for yourself. You've certainly obtained +"service" (computer service) and a "thing of value" (the software). +Suppose you tell a dozen friends about your swiped password, +and let them use it, too. Now you're "trafficking in unauthorized +access devices." And when the Prophet, a member of the Legion of Doom, +passed a stolen telephone company document to Knight Lightning +at Phrack magazine, they were both charged under Sec. 1029! + +There are two limitations on Section 1029. First, the offense must +"affect interstate or foreign commerce" in order to become a matter +of federal jurisdiction. The term "affecting commerce" is not well defined; +but you may take it as a given that the Secret Service can take an interest +if you've done most anything that happens to cross a state line. +State and local police can be touchy about their jurisdictions, +and can sometimes be mulish when the feds show up. But when it comes +to computer-crime, the local police are pathetically grateful +for federal help--in fact they complain that they can't get enough of it. +If you're stealing long-distance service, you're almost certainly crossing +state lines, and you're definitely "affecting the interstate commerce" +of the telcos. And if you're abusing credit cards by ordering stuff +out of glossy catalogs from, say, Vermont, you're in for it. + +The second limitation is money. As a rule, the feds don't pursue +penny-ante offenders. Federal judges will dismiss cases that appear +to waste their time. Federal crimes must be serious; Section 1029 +specifies a minimum loss of a thousand dollars. + +We now come to the very next section of Title 18, which is Section 1030, +"Fraud and related activity in connection with computers." This statute +gives the Secret Service direct jurisdiction over acts of computer intrusion. +On the face of it, the Secret Service would now seem to command the field. +Section 1030, however, is nowhere near so ductile as Section 1029. + +The first annoyance is Section 1030(d), which reads: + +"(d) The United States Secret Service shall, +IN ADDITION TO ANY OTHER AGENCY HAVING SUCH AUTHORITY, +have the authority to investigate offenses under this section. +Such authority of the United States Secret Service shall be +exercised in accordance with an agreement which shall be entered +into by the Secretary of the Treasury AND THE ATTORNEY GENERAL." +(Author's italics.) [Represented by capitals.] + +The Secretary of the Treasury is the titular head of the Secret Service, +while the Attorney General is in charge of the FBI. In Section (d), +Congress shrugged off responsibility for the computer-crime turf-battle +between the Service and the Bureau, and made them fight it out all +by themselves. The result was a rather dire one for the Secret Service, +for the FBI ended up with exclusive jurisdiction over computer break-ins +having to do with national security, foreign espionage, federally insured +banks, and U.S. military bases, while retaining joint jurisdiction over +all the other computer intrusions. Essentially, when it comes to Section 1030, +the FBI not only gets the real glamor stuff for itself, but can peer over the +shoulder of the Secret Service and barge in to meddle whenever it suits them. + +The second problem has to do with the dicey term +"Federal interest computer." Section 1030(a)(2) +makes it illegal to "access a computer without authorization" +if that computer belongs to a financial institution or an issuer +of credit cards (fraud cases, in other words). Congress was quite +willing to give the Secret Service jurisdiction over +money-transferring computers, but Congress balked at +letting them investigate any and all computer intrusions. +Instead, the USSS had to settle for the money machines +and the "Federal interest computers." A "Federal interest computer" +is a computer which the government itself owns, or is using. +Large networks of interstate computers, linked over state lines, +are also considered to be of "Federal interest." (This notion of +"Federal interest" is legally rather foggy and has never been +clearly defined in the courts. The Secret Service has never yet +had its hand slapped for investigating computer break-ins that were NOT +of "Federal interest," but conceivably someday this might happen.) + +So the Secret Service's authority over "unauthorized access" +to computers covers a lot of territory, but by no means the +whole ball of cyberspatial wax. If you are, for instance, +a LOCAL computer retailer, or the owner of a LOCAL bulletin +board system, then a malicious LOCAL intruder can break in, +crash your system, trash your files and scatter viruses, +and the U.S. Secret Service cannot do a single thing about it. + +At least, it can't do anything DIRECTLY. But the Secret Service +will do plenty to help the local people who can. + +The FBI may have dealt itself an ace off the bottom of the deck +when it comes to Section 1030; but that's not the whole story; +that's not the street. What's Congress thinks is one thing, +and Congress has been known to change its mind. The REAL +turf-struggle is out there in the streets where it's happening. +If you're a local street-cop with a computer problem, +the Secret Service wants you to know where you can find +the real expertise. While the Bureau crowd are off having +their favorite shoes polished--(wing-tips)--and making derisive +fun of the Service's favorite shoes--("pansy-ass tassels")-- +the tassel-toting Secret Service has a crew of ready-and-able +hacker-trackers installed in the capital of every state in the Union. +Need advice? They'll give you advice, or at least point you in +the right direction. Need training? They can see to that, too. + +If you're a local cop and you call in the FBI, the FBI +(as is widely and slanderously rumored) will order you around +like a coolie, take all the credit for your busts, +and mop up every possible scrap of reflected glory. +The Secret Service, on the other hand, doesn't brag a lot. +They're the quiet types. VERY quiet. Very cool. Efficient. +High-tech. Mirrorshades, icy stares, radio ear-plugs, +an Uzi machine-pistol tucked somewhere in that well-cut jacket. +American samurai, sworn to give their lives to protect our President. +"The granite agents." Trained in martial arts, absolutely fearless. +Every single one of 'em has a top-secret security clearance. +Something goes a little wrong, you're not gonna hear any whining +and moaning and political buck-passing out of these guys. + +The facade of the granite agent is not, of course, the reality. +Secret Service agents are human beings. And the real glory +in Service work is not in battling computer crime--not yet, +anyway--but in protecting the President. The real glamour +of Secret Service work is in the White House Detail. +If you're at the President's side, then the kids and the wife +see you on television; you rub shoulders with the most powerful +people in the world. That's the real heart of Service work, +the number one priority. More than one computer investigation +has stopped dead in the water when Service agents vanished at +the President's need. + +There's romance in the work of the Service. The intimate access +to circles of great power; the esprit-de-corps of a highly trained +and disciplined elite; the high responsibility of defending the +Chief Executive; the fulfillment of a patriotic duty. And as police +work goes, the pay's not bad. But there's squalor in Service work, too. +You may get spat upon by protesters howling abuse--and if they get violent, +if they get too close, sometimes you have to knock one of them down-- +discreetly. + +The real squalor in Service work is drudgery such as "the quarterlies," +traipsing out four times a year, year in, year out, to interview the various +pathetic wretches, many of them in prisons and asylums, who have seen fit +to threaten the President's life. And then there's the grinding stress +of searching all those faces in the endless bustling crowds, looking for +hatred, looking for psychosis, looking for the tight, nervous face +of an Arthur Bremer, a Squeaky Fromme, a Lee Harvey Oswald. +It's watching all those grasping, waving hands for sudden movements, +while your ears strain at your radio headphone for the long-rehearsed +cry of "Gun!" + +It's poring, in grinding detail, over the biographies of every rotten +loser who ever shot at a President. It's the unsung work of the +Protective Research Section, who study scrawled, anonymous death threats +with all the meticulous tools of anti-forgery techniques. + +And it's maintaining the hefty computerized files on anyone +who ever threatened the President's life. Civil libertarians +have become increasingly concerned at the Government's use +of computer files to track American citizens--but the +Secret Service file of potential Presidential assassins, +which has upward of twenty thousand names, rarely causes +a peep of protest. If you EVER state that you intend to +kill the President, the Secret Service will want to know +and record who you are, where you are, what you are, +and what you're up to. If you're a serious threat-- +if you're officially considered "of protective interest"-- +then the Secret Service may well keep tabs on you +for the rest of your natural life. + +Protecting the President has first call on all the Service's resources. +But there's a lot more to the Service's traditions and history than +standing guard outside the Oval Office. + +The Secret Service is the nation's oldest general federal +law-enforcement agency. Compared to the Secret Service, +the FBI are new-hires and the CIA are temps. The Secret Service +was founded 'way back in 1865, at the suggestion of Hugh McCulloch, +Abraham Lincoln's Secretary of the Treasury. McCulloch wanted +a specialized Treasury police to combat counterfeiting. +Abraham Lincoln agreed that this seemed a good idea, and, +with a terrible irony, Abraham Lincoln was shot that +very night by John Wilkes Booth. + +The Secret Service originally had nothing to do with protecting Presidents. +They didn't take this on as a regular assignment until after the Garfield +assassination in 1881. And they didn't get any Congressional money for it +until President McKinley was shot in 1901. The Service was originally +designed for one purpose: destroying counterfeiters. + +# + +There are interesting parallels between the Service's +nineteenth-century entry into counterfeiting, +and America's twentieth-century entry into computer-crime. + +In 1865, America's paper currency was a terrible muddle. +Security was drastically bad. Currency was printed on the spot +by local banks in literally hundreds of different designs. +No one really knew what the heck a dollar bill was supposed to look like. +Bogus bills passed easily. If some joker told you that a one-dollar bill +from the Railroad Bank of Lowell, Massachusetts had a woman leaning on +a shield, with a locomotive, a cornucopia, a compass, various agricultural +implements, a railroad bridge, and some factories, then you pretty much had +to take his word for it. (And in fact he was telling the truth!) + +SIXTEEN HUNDRED local American banks designed and printed their own +paper currency, and there were no general standards for security. +Like a badly guarded node in a computer network, badly designed bills +were easy to fake, and posed a security hazard for the entire monetary system. + +No one knew the exact extent of the threat to the currency. +There were panicked estimates that as much as a third of +the entire national currency was faked. Counterfeiters-- +known as "boodlers" in the underground slang of the time-- +were mostly technically skilled printers who had gone to the bad. +Many had once worked printing legitimate currency. +Boodlers operated in rings and gangs. Technical experts +engraved the bogus plates--commonly in basements in New York City. +Smooth confidence men passed large wads of high-quality, +high-denomination fakes, including the really sophisticated stuff-- +government bonds, stock certificates, and railway shares. +Cheaper, botched fakes were sold or sharewared to low-level +gangs of boodler wannabes. (The really cheesy lowlife boodlers +merely upgraded real bills by altering face values, +changing ones to fives, tens to hundreds, and so on.) + +The techniques of boodling were little-known and regarded +with a certain awe by the mid- nineteenth-century public. +The ability to manipulate the system for rip-off seemed +diabolically clever. As the skill and daring of the +boodlers increased, the situation became intolerable. +The federal government stepped in, and began offering +its own federal currency, which was printed in fancy green ink, +but only on the back--the original "greenbacks." And at first, +the improved security of the well-designed, well-printed +federal greenbacks seemed to solve the problem; but then +the counterfeiters caught on. Within a few years things were +worse than ever: a CENTRALIZED system where ALL security was bad! + +The local police were helpless. The Government tried offering +blood money to potential informants, but this met with little success. +Banks, plagued by boodling, gave up hope of police help and hired +private security men instead. Merchants and bankers queued up +by the thousands to buy privately-printed manuals on currency security, +slim little books like Laban Heath's INFALLIBLE GOVERNMENT +COUNTERFEIT DETECTOR. The back of the book offered Laban Heath's +patent microscope for five bucks. + +Then the Secret Service entered the picture. The first agents +were a rough and ready crew. Their chief was one William P. Wood, +a former guerilla in the Mexican War who'd won a reputation busting +contractor fraudsters for the War Department during the Civil War. +Wood, who was also Keeper of the Capital Prison, had a sideline +as a counterfeiting expert, bagging boodlers for the federal bounty money. + +Wood was named Chief of the new Secret Service in July 1865. +There were only ten Secret Service agents in all: Wood himself, +a handful who'd worked for him in the War Department, and a few +former private investigators--counterfeiting experts--whom Wood +had won over to public service. (The Secret Service of 1865 was +much the size of the Chicago Computer Fraud Task Force or the +Arizona Racketeering Unit of 1990.) These ten "Operatives" +had an additional twenty or so "Assistant Operatives" and "Informants." +Besides salary and per diem, each Secret Service employee received +a whopping twenty-five dollars for each boodler he captured. + +Wood himself publicly estimated that at least HALF of America's currency +was counterfeit, a perhaps pardonable perception. Within a year the +Secret Service had arrested over 200 counterfeiters. They busted about +two hundred boodlers a year for four years straight. + +Wood attributed his success to travelling fast and light, hitting the +bad-guys hard, and avoiding bureaucratic baggage. "Because my raids +were made without military escort and I did not ask the assistance +of state officers, I surprised the professional counterfeiter." + +Wood's social message to the once-impudent boodlers bore an eerie ring +of Sundevil: "It was also my purpose to convince such characters that +it would no longer be healthy for them to ply their vocation without +being handled roughly, a fact they soon discovered." + +William P. Wood, the Secret Service's guerilla pioneer, +did not end well. He succumbed to the lure of aiming for +the really big score. The notorious Brockway Gang of New York City, +headed by William E. Brockway, the "King of the Counterfeiters," +had forged a number of government bonds. They'd passed these +brilliant fakes on the prestigious Wall Street investment +firm of Jay Cooke and Company. The Cooke firm were frantic +and offered a huge reward for the forgers' plates. + +Laboring diligently, Wood confiscated the plates +(though not Mr. Brockway) and claimed the reward. +But the Cooke company treacherously reneged. +Wood got involved in a down-and-dirty lawsuit +with the Cooke capitalists. Wood's boss, +Secretary of the Treasury McCulloch, felt that +Wood's demands for money and glory were unseemly, +and even when the reward money finally came through, +McCulloch refused to pay Wood anything. +Wood found himself mired in a seemingly endless +round of federal suits and Congressional lobbying. + +Wood never got his money. And he lost his job to boot. +He resigned in 1869. + +Wood's agents suffered, too. On May 12, 1869, the second Chief +of the Secret Service took over, and almost immediately fired +most of Wood's pioneer Secret Service agents: Operatives, +Assistants and Informants alike. The practice of receiving $25 +per crook was abolished. And the Secret Service began the long, +uncertain process of thorough professionalization. + +Wood ended badly. He must have felt stabbed in the back. +In fact his entire organization was mangled. + +On the other hand, William P. Wood WAS the first head of the Secret Service. +William Wood was the pioneer. People still honor his name. Who remembers +the name of the SECOND head of the Secret Service? + +As for William Brockway (also known as "Colonel Spencer"), +he was finally arrested by the Secret Service in 1880. +He did five years in prison, got out, and was still boodling +at the age of seventy-four. + +# + +Anyone with an interest in Operation Sundevil-- +or in American computer-crime generally-- +could scarcely miss the presence of Gail Thackeray, +Assistant Attorney General of the State of Arizona. +Computer-crime training manuals often cited +Thackeray's group and her work; she was the +highest-ranking state official to specialize +in computer-related offenses. Her name had been +on the Sundevil press release (though modestly ranked +well after the local federal prosecuting attorney and +the head of the Phoenix Secret Service office). + +As public commentary, and controversy, began to mount +about the Hacker Crackdown, this Arizonan state official +began to take a higher and higher public profile. +Though uttering almost nothing specific about +the Sundevil operation itself, she coined some +of the most striking soundbites of the growing propaganda war: +"Agents are operating in good faith, and I don't think +you can say that for the hacker community," was one. +Another was the memorable "I am not a mad dog prosecutor" +(Houston Chronicle, Sept 2, 1990.) In the meantime, +the Secret Service maintained its usual extreme discretion; +the Chicago Unit, smarting from the backlash +of the Steve Jackson scandal, had gone completely to earth. + +As I collated my growing pile of newspaper clippings, +Gail Thackeray ranked as a comparative fount of public +knowledge on police operations. + +I decided that I had to get to know Gail Thackeray. +I wrote to her at the Arizona Attorney General's Office. +Not only did she kindly reply to me, but, to my astonishment, +she knew very well what "cyberpunk" science fiction was. + +Shortly after this, Gail Thackeray lost her job. +And I temporarily misplaced my own career as +a science-fiction writer, to become a full-time +computer-crime journalist. In early March, 1991, +I flew to Phoenix, Arizona, to interview Gail Thackeray +for my book on the hacker crackdown. + +# + +"Credit cards didn't used to cost anything to get," +says Gail Thackeray. "Now they cost forty bucks-- +and that's all just to cover the costs from RIP-OFF ARTISTS." + +Electronic nuisance criminals are parasites. +One by one they're not much harm, no big deal. +But they never come just one by one. They come in swarms, +heaps, legions, sometimes whole subcultures. And they bite. +Every time we buy a credit card today, we lose a little financial +vitality to a particular species of bloodsucker. + +What, in her expert opinion, are the worst forms of electronic crime, +I ask, consulting my notes. Is it--credit card fraud? Breaking into +ATM bank machines? Phone-phreaking? Computer intrusions? +Software viruses? Access-code theft? Records tampering? +Software piracy? Pornographic bulletin boards? +Satellite TV piracy? Theft of cable service? +It's a long list. By the time I reach the end +of it I feel rather depressed. + +"Oh no," says Gail Thackeray, leaning forward over the table, +her whole body gone stiff with energetic indignation, +"the biggest damage is telephone fraud. Fake sweepstakes, +fake charities. Boiler-room con operations. You could pay off +the national debt with what these guys steal. . . . +They target old people, they get hold of credit ratings +and demographics, they rip off the old and the weak." +The words come tumbling out of her. + +It's low-tech stuff, your everyday boiler-room fraud. +Grifters, conning people out of money over the phone, +have been around for decades. This is where the word "phony" came from! + +It's just that it's so much EASIER now, horribly facilitated by advances +in technology and the byzantine structure of the modern phone system. +The same professional fraudsters do it over and over, Thackeray tells me, +they hide behind dense onion-shells of fake companies. . . fake holding +corporations nine or ten layers deep, registered all over the map. +They get a phone installed under a false name in an empty safe-house. +And then they call-forward everything out of that phone to yet +another phone, a phone that may even be in another STATE. +And they don't even pay the charges on their phones; +after a month or so, they just split; set up somewhere else +in another Podunkville with the same seedy crew of veteran phone-crooks. +They buy or steal commercial credit card reports, slap them on the PC, +have a program pick out people over sixty-five who pay a lot to charities. +A whole subculture living off this, merciless folks on the con. + +"The `light-bulbs for the blind' people," Thackeray muses, +with a special loathing. "There's just no end to them." + +We're sitting in a downtown diner in Phoenix, Arizona. +It's a tough town, Phoenix. A state capital seeing some hard times. +Even to a Texan like myself, Arizona state politics seem rather baroque. +There was, and remains, endless trouble over the Martin Luther King holiday, +the sort of stiff-necked, foot-shooting incident for which Arizona politics +seem famous. There was Evan Mecham, the eccentric Republican millionaire +governor who was impeached, after reducing state government to a +ludicrous shambles. Then there was the national Keating scandal, +involving Arizona savings and loans, in which both of Arizona's +U.S. senators, DeConcini and McCain, played sadly prominent roles. + +And the very latest is the bizarre AzScam case, +in which state legislators were videotaped, +eagerly taking cash from an informant of the Phoenix city +police department, who was posing as a Vegas mobster. + +"Oh," says Thackeray cheerfully. "These people are amateurs here, +they thought they were finally getting to play with the big boys. +They don't have the least idea how to take a bribe! +It's not institutional corruption. It's not like back in Philly." + +Gail Thackeray was a former prosecutor in Philadelphia. +Now she's a former assistant attorney general of the State of Arizona. +Since moving to Arizona in 1986, she had worked under the aegis +of Steve Twist, her boss in the Attorney General's office. +Steve Twist wrote Arizona's pioneering computer crime laws +and naturally took an interest in seeing them enforced. +It was a snug niche, and Thackeray's Organized Crime and +Racketeering Unit won a national reputation for ambition +and technical knowledgeability. . . . Until the latest +election in Arizona. Thackeray's boss ran for the top +job, and lost. The victor, the new Attorney General, +apparently went to some pains to eliminate the bureaucratic +traces of his rival, including his pet group--Thackeray's group. +Twelve people got their walking papers. + +Now Thackeray's painstakingly assembled computer lab +sits gathering dust somewhere in the glass-and-concrete +Attorney General's HQ on 1275 Washington Street. +Her computer-crime books, her painstakingly garnered +back issues of phreak and hacker zines, all bought +at her own expense--are piled in boxes somewhere. +The State of Arizona is simply not particularly +interested in electronic racketeering at the moment. + +At the moment of our interview, Gail Thackeray, +officially unemployed, is working out of the county +sheriff's office, living on her savings, and prosecuting +several cases--working 60-hour weeks, just as always-- +for no pay at all. "I'm trying to train people," +she mutters. + +Half her life seems to be spent training people--merely pointing out, +to the naive and incredulous (such as myself) that this stuff +is ACTUALLY GOING ON OUT THERE. It's a small world, computer crime. +A young world. Gail Thackeray, a trim blonde Baby-Boomer who favors +Grand Canyon white-water rafting to kill some slow time, +is one of the world's most senior, most veteran "hacker-trackers." +Her mentor was Donn Parker, the California think-tank theorist +who got it all started `way back in the mid-70s, the "grandfather +of the field," "the great bald eagle of computer crime." + +And what she has learned, Gail Thackeray teaches. Endlessly. +Tirelessly. To anybody. To Secret Service agents and state police, +at the Glynco, Georgia federal training center. To local police, +on "roadshows" with her slide projector and notebook. +To corporate security personnel. To journalists. To parents. + +Even CROOKS look to Gail Thackeray for advice. +Phone-phreaks call her at the office. They know very +well who she is. They pump her for information +on what the cops are up to, how much they know. +Sometimes whole CROWDS of phone phreaks, +hanging out on illegal conference calls, will call Gail +Thackeray up. They taunt her. And, as always, +they boast. Phone-phreaks, real stone phone-phreaks, +simply CANNOT SHUT UP. They natter on for hours. + +Left to themselves, they mostly talk about the intricacies +of ripping-off phones; it's about as interesting as listening +to hot-rodders talk about suspension and distributor-caps. +They also gossip cruelly about each other. And when talking +to Gail Thackeray, they incriminate themselves. "I have tapes," +Thackeray says coolly. + +Phone phreaks just talk like crazy. "Dial-Tone" out in Alabama +has been known to spend half-an-hour simply reading stolen +phone-codes aloud into voice-mail answering machines. +Hundreds, thousands of numbers, recited in a monotone, +without a break--an eerie phenomenon. When arrested, +it's a rare phone phreak who doesn't inform at endless length +on everybody he knows. + +Hackers are no better. What other group of criminals, +she asks rhetorically, publishes newsletters and holds conventions? +She seems deeply nettled by the sheer brazenness of this behavior, +though to an outsider, this activity might make one wonder +whether hackers should be considered "criminals" at all. +Skateboarders have magazines, and they trespass a lot. +Hot rod people have magazines and they break speed limits +and sometimes kill people. . . . + +I ask her whether it would be any loss to society if phone phreaking +and computer hacking, as hobbies, simply dried up and blew away, +so that nobody ever did it again. + +She seems surprised. "No," she says swiftly. "Maybe a little. . . +in the old days. . .the MIT stuff. . . . But there's a lot of wonderful, +legal stuff you can do with computers now, you don't have to break into +somebody else's just to learn. You don't have that excuse. +You can learn all you like." + +Did you ever hack into a system? I ask. + +The trainees do it at Glynco. Just to demonstrate system vulnerabilities. +She's cool to the notion. Genuinely indifferent. + +"What kind of computer do you have?" + +"A Compaq 286LE," she mutters. + +"What kind do you WISH you had?" + +At this question, the unmistakable light of true hackerdom flares in +Gail Thackeray's eyes. She becomes tense, animated, the words pour out: +"An Amiga 2000 with an IBM card and Mac emulation! The most common hacker +machines are Amigas and Commodores. And Apples." If she had the Amiga, +she enthuses, she could run a whole galaxy of seized computer-evidence disks +on one convenient multifunctional machine. A cheap one, too. Not like the +old Attorney General lab, where they had an ancient CP/M machine, +assorted Amiga flavors and Apple flavors, a couple IBMS, all the +utility software. . .but no Commodores. The workstations down +at the Attorney General's are Wang dedicated word-processors. +Lame machines tied in to an office net--though at least they get +on- line to the Lexis and Westlaw legal data services. + +I don't say anything. I recognize the syndrome, though. +This computer-fever has been running through segments of +our society for years now. It's a strange kind of lust: +K-hunger, Meg-hunger; but it's a shared disease; +it can kill parties dead, as conversation spirals into +the deepest and most deviant recesses of software releases +and expensive peripherals. . . . The mark of the hacker beast. +I have it too. The whole "electronic community," whatever the hell +that is, has it. Gail Thackeray has it. Gail Thackeray is a hacker cop. +My immediate reaction is a strong rush of indignant pity: +WHY DOESN'T SOMEBODY BUY THIS WOMAN HER AMIGA?! +It's not like she's asking for a Cray X-MP +supercomputer mainframe; an Amiga's a sweet little +cookie-box thing. We're losing zillions in organized fraud; +prosecuting and defending a single hacker case in court can cost +a hundred grand easy. How come nobody can come up with four lousy grand +so this woman can do her job? For a hundred grand we could buy every +computer cop in America an Amiga. There aren't that many of 'em. + +Computers. The lust, the hunger, for computers. +The loyalty they inspire, the intense sense of possessiveness. +The culture they have bred. I myself am sitting in downtown Phoenix, +Arizona because it suddenly occurred to me that the police might-- +just MIGHT--come and take away my computer. The prospect of this, +the mere IMPLIED THREAT, was unbearable. It literally changed my life. +It was changing the lives of many others. Eventually it would change +everybody's life. + +Gail Thackeray was one of the top computer-crime people in America. +And I was just some novelist, and yet I had a better computer than hers. +PRACTICALLY EVERYBODY I KNEW had a better computer than Gail Thackeray +and her feeble laptop 286. It was like sending the sheriff in to clean +up Dodge City and arming her with a slingshot cut from an old rubber tire. + +But then again, you don't need a howitzer to enforce the law. +You can do a lot just with a badge. With a badge alone, +you can basically wreak havoc, take a terrible vengeance on wrongdoers. +Ninety percent of "computer crime investigation" is just "crime investigation:" +names, places, dossiers, modus operandi, search warrants, victims, +complainants, informants. . . . + +What will computer crime look like in ten years? Will it get better? +Did "Sundevil" send 'em reeling back in confusion? + +It'll be like it is now, only worse, she tells me with perfect conviction. +Still there in the background, ticking along, changing with the times: +the criminal underworld. It'll be like drugs are. Like our problems +with alcohol. All the cops and laws in the world never solved our problems +with alcohol. If there's something people want, a certain percentage +of them are just going to take it. Fifteen percent of the populace +will never steal. Fifteen percent will steal most anything not nailed down. +The battle is for the hearts and minds of the remaining seventy percent. + +And criminals catch on fast. If there's not "too steep a learning curve"-- +if it doesn't require a baffling amount of expertise and practice-- +then criminals are often some of the first through the gate of a +new technology. Especially if it helps them to hide. +They have tons of cash, criminals. The new communications tech-- +like pagers, cellular phones, faxes, Federal Express--were pioneered +by rich corporate people, and by criminals. In the early years +of pagers and beepers, dope dealers were so enthralled this technology +that owing a beeper was practically prima facie evidence of cocaine dealing. +CB radio exploded when the speed limit hit 55 and breaking the highway law +became a national pastime. Dope dealers send cash by Federal Express, +despite, or perhaps BECAUSE OF, the warnings in FedEx offices that tell you +never to try this. Fed Ex uses X-rays and dogs on their mail, +to stop drug shipments. That doesn't work very well. + +Drug dealers went wild over cellular phones. +There are simple methods of faking ID on cellular phones, +making the location of the call mobile, free of charge, +and effectively untraceable. Now victimized cellular +companies routinely bring in vast toll-lists of calls +to Colombia and Pakistan. + +Judge Greene's fragmentation of the phone company +is driving law enforcement nuts. Four thousand +telecommunications companies. Fraud skyrocketing. +Every temptation in the world available with a phone +and a credit card number. Criminals untraceable. +A galaxy of "new neat rotten things to do." + +If there were one thing Thackeray would like to have, +it would be an effective legal end-run through this new +fragmentation minefield. + +It would be a new form of electronic search warrant, +an "electronic letter of marque" to be issued by a judge. +It would create a new category of "electronic emergency." +Like a wiretap, its use would be rare, but it would cut +across state lines and force swift cooperation from all concerned. +Cellular, phone, laser, computer network, PBXes, AT&T, Baby Bells, +long-distance entrepreneurs, packet radio. Some document, +some mighty court-order, that could slice through four thousand +separate forms of corporate red-tape, and get her at once to +the source of calls, the source of email threats and viruses, +the sources of bomb threats, kidnapping threats. "From now on," +she says, "the Lindbergh baby will always die." + +Something that would make the Net sit still, if only for a moment. +Something that would get her up to speed. Seven league boots. +That's what she really needs. "Those guys move in nanoseconds +and I'm on the Pony Express." + +And then, too, there's the coming international angle. +Electronic crime has never been easy to localize, +to tie to a physical jurisdiction. And phone-phreaks +and hackers loathe boundaries, they jump them whenever they can. +The English. The Dutch. And the Germans, especially the ubiquitous +Chaos Computer Club. The Australians. They've all learned phone-phreaking +from America. It's a growth mischief industry. The multinational +networks are global, but governments and the police simply aren't. +Neither are the laws. Or the legal frameworks for citizen protection. + +One language is global, though--English. Phone phreaks speak English; +it's their native tongue even if they're Germans. English may have started +in England but now it's the Net language; it might as well be called "CNNese." + +Asians just aren't much into phone phreaking. They're the world masters +at organized software piracy. The French aren't into phone-phreaking either. +The French are into computerized industrial espionage. + +In the old days of the MIT righteous hackerdom, crashing systems +didn't hurt anybody. Not all that much, anyway. Not permanently. +Now the players are more venal. Now the consequences are worse. +Hacking will begin killing people soon. Already there are methods +of stacking calls onto 911 systems, annoying the police, and possibly +causing the death of some poor soul calling in with a genuine emergency. +Hackers in Amtrak computers, or air-traffic control computers, will kill +somebody someday. Maybe a lot of people. Gail Thackeray expects it. + +And the viruses are getting nastier. The "Scud" virus is the latest one out. +It wipes hard-disks. + +According to Thackeray, the idea that phone-phreaks are Robin Hoods is a fraud. +They don't deserve this repute. Basically, they pick on the weak. AT&T now +protects itself with the fearsome ANI (Automatic Number Identification) +trace capability. When AT&T wised up and tightened security generally, +the phreaks drifted into the Baby Bells. The Baby Bells lashed out in 1989 +and 1990, so the phreaks switched to smaller long-distance entrepreneurs. +Today, they are moving into locally owned PBXes and voice-mail systems, +which are full of security holes, dreadfully easy to hack. These victims +aren't the moneybags Sheriff of Nottingham or Bad King John, but small groups +of innocent people who find it hard to protect themselves, and who really +suffer from these depredations. Phone phreaks pick on the weak. They do it +for power. If it were legal, they wouldn't do it. They don't want service, +or knowledge, they want the thrill of power-tripping. There's plenty of +knowledge or service around if you're willing to pay. Phone phreaks don't pay, +they steal. It's because it is illegal that it feels like power, +that it gratifies their vanity. + +I leave Gail Thackeray with a handshake at the door of her office building-- +a vast International-Style office building downtown. The Sheriff's office +is renting part of it. I get the vague impression that quite a lot of the +building is empty--real estate crash. + +In a Phoenix sports apparel store, in a downtown mall, I meet +the "Sun Devil" himself. He is the cartoon mascot of +Arizona State University, whose football stadium, "Sundevil," +is near the local Secret Service HQ--hence the name Operation Sundevil. +The Sun Devil himself is named "Sparky." Sparky the Sun Devil is maroon +and bright yellow, the school colors. Sparky brandishes a three-tined +yellow pitchfork. He has a small mustache, pointed ears, a barbed tail, +and is dashing forward jabbing the air with the pitchfork, +with an expression of devilish glee. + +Phoenix was the home of Operation Sundevil. The Legion of Doom +ran a hacker bulletin board called "The Phoenix Project." +An Australian hacker named "Phoenix" once burrowed through +the Internet to attack Cliff Stoll, then bragged and boasted +about it to The New York Times. This net of coincidence +is both odd and meaningless. + +The headquarters of the Arizona Attorney General, Gail Thackeray's +former workplace, is on 1275 Washington Avenue. Many of the downtown +streets in Phoenix are named after prominent American presidents: +Washington, Jefferson, Madison. . . . + +After dark, all the employees go home to their suburbs. +Washington, Jefferson and Madison--what would be the +Phoenix inner city, if there were an inner city in this +sprawling automobile-bred town--become the haunts +of transients and derelicts. The homeless. The sidewalks +along Washington are lined with orange trees. +Ripe fallen fruit lies scattered like croquet balls +on the sidewalks and gutters. No one seems to be eating them. +I try a fresh one. It tastes unbearably bitter. + +The Attorney General's office, built in 1981 during the +Babbitt administration, is a long low two-story building +of white cement and wall-sized sheets of curtain-glass. +Behind each glass wall is a lawyer's office, quite open +and visible to anyone strolling by. Across the street +is a dour government building labelled simply ECONOMIC SECURITY, +something that has not been in great supply in the American +Southwest lately. + +The offices are about twelve feet square. They feature +tall wooden cases full of red-spined lawbooks; +Wang computer monitors; telephones; Post-it notes galore. +Also framed law diplomas and a general excess of bad +Western landscape art. Ansel Adams photos are a big favorite, +perhaps to compensate for the dismal specter of the parking lot, +two acres of striped black asphalt, which features gravel landscaping +and some sickly-looking barrel cacti. + +It has grown dark. Gail Thackeray has told me that the people +who work late here, are afraid of muggings in the parking lot. +It seems cruelly ironic that a woman tracing electronic racketeers +across the interstate labyrinth of Cyberspace should fear an assault +by a homeless derelict in the parking lot of her own workplace. + +Perhaps this is less than coincidence. Perhaps these two seemingly +disparate worlds are somehow generating one another. The poor and +disenfranchised take to the streets, while the rich and computer-equipped, +safe in their bedrooms, chatter over their modems. Quite often the derelicts +kick the glass out and break in to the lawyers' offices, if they see something +they need or want badly enough. + +I cross the parking lot to the street behind the Attorney General's office. +A pair of young tramps are bedding down on flattened sheets of cardboard, +under an alcove stretching over the sidewalk. One tramp wears a +glitter-covered T-shirt reading "CALIFORNIA" in Coca-Cola cursive. +His nose and cheeks look chafed and swollen; they glisten with +what seems to be Vaseline. The other tramp has a ragged long-sleeved +shirt and lank brown hair parted in the middle. They both wear blue jeans +coated in grime. They are both drunk. + +"You guys crash here a lot?" I ask them. + +They look at me warily. I am wearing black jeans, a black pinstriped +suit jacket and a black silk tie. I have odd shoes and a funny haircut. + +"It's our first time here," says the red-nosed tramp unconvincingly. +There is a lot of cardboard stacked here. More than any two people could use. + +"We usually stay at the Vinnie's down the street," says the brown-haired tramp, +puffing a Marlboro with a meditative air, as he sprawls with his head on +a blue nylon backpack. "The Saint Vincent's." + +"You know who works in that building over there?" I ask, pointing. + +The brown-haired tramp shrugs. "Some kind of attorneys, it says." + +We urge one another to take it easy. I give them five bucks. + +A block down the street I meet a vigorous workman who is wheeling along +some kind of industrial trolley; it has what appears to be a tank of +propane on it. + +We make eye contact. We nod politely. I walk past him. "Hey! +Excuse me sir!" he says. + +"Yes?" I say, stopping and turning. + +"Have you seen," the guy says rapidly, "a black guy, about 6'7", +scars on both his cheeks like this--" he gestures-- "wears a +black baseball cap on backwards, wandering around here anyplace?" + +"Sounds like I don't much WANT to meet him," I say. + +"He took my wallet," says my new acquaintance. +"Took it this morning. Y'know, some people would be +SCARED of a guy like that. But I'm not scared. +I'm from Chicago. I'm gonna hunt him down. +We do things like that in Chicago." + +"Yeah?" + +"I went to the cops and now he's got an APB out on his ass," +he says with satisfaction. "You run into him, you let me know." + +"Okay," I say. "What is your name, sir?" + +"Stanley. . . ." + +"And how can I reach you?" + +"Oh," Stanley says, in the same rapid voice, +"you don't have to reach, uh, me. +You can just call the cops. Go straight to the cops." +He reaches into a pocket and pulls out a greasy piece of pasteboard. +"See, here's my report on him." + +I look. The "report," the size of an index card, is labelled PRO-ACT: +Phoenix Residents Opposing Active Crime Threat. . . . or is it +Organized Against Crime Threat? In the darkening street it's hard +to read. Some kind of vigilante group? Neighborhood watch? +I feel very puzzled. + +"Are you a police officer, sir?" + +He smiles, seems very pleased by the question. + +"No," he says. + +"But you are a `Phoenix Resident?'" + +"Would you believe a homeless person," Stanley says. + +"Really? But what's with the. . . ." For the first time I take a close look +at Stanley's trolley. It's a rubber-wheeled thing of industrial metal, +but the device I had mistaken for a tank of propane is in fact a water-cooler. +Stanley also has an Army duffel-bag, stuffed tight as a sausage with clothing +or perhaps a tent, and, at the base of his trolley, a cardboard box and a +battered leather briefcase. + +"I see," I say, quite at a loss. For the first time I notice that Stanley +has a wallet. He has not lost his wallet at all. It is in his back pocket +and chained to his belt. It's not a new wallet. It seems to have seen +a lot of wear. + +"Well, you know how it is, brother," says Stanley. +Now that I know that he is homeless--A POSSIBLE +THREAT--my entire perception of him has changed +in an instant. His speech, which once seemed just +bright and enthusiastic, now seems to have a +dangerous tang of mania. "I have to do this!" +he assures me. "Track this guy down. . . . +It's a thing I do. . . you know. . .to keep myself together!" +He smiles, nods, lifts his trolley by its decaying rubber handgrips. + +"Gotta work together, y'know," Stanley booms, his face alight +with cheerfulness, "the police can't do everything!" +The gentlemen I met in my stroll in downtown Phoenix +are the only computer illiterates in this book. +To regard them as irrelevant, however, would be a grave mistake. + +As computerization spreads across society, the populace at large +is subjected to wave after wave of future shock. But, as a +necessary converse, the "computer community" itself is subjected +to wave after wave of incoming computer illiterates. +How will those currently enjoying America's digital bounty regard, +and treat, all this teeming refuse yearning to breathe free? +Will the electronic frontier be another Land of Opportunity-- +or an armed and monitored enclave, where the disenfranchised +snuggle on their cardboard at the locked doors of our houses of justice? + +Some people just don't get along with computers. They can't read. +They can't type. They just don't have it in their heads to master +arcane instructions in wirebound manuals. Somewhere, the process +of computerization of the populace will reach a limit. Some people-- +quite decent people maybe, who might have thrived in any other situation-- +will be left irretrievably outside the bounds. What's to be done with +these people, in the bright new shiny electroworld? How will they +be regarded, by the mouse-whizzing masters of cyberspace? With contempt? +Indifference? Fear? + +In retrospect, it astonishes me to realize how quickly poor Stanley +became a perceived threat. Surprise and fear are closely allied feelings. +And the world of computing is full of surprises. + +I met one character in the streets of Phoenix whose role in this book +is supremely and directly relevant. That personage was Stanley's giant +thieving scarred phantom. This phantasm is everywhere in this book. +He is the specter haunting cyberspace. + +Sometimes he's a maniac vandal ready to smash the phone system +for no sane reason at all. Sometimes he's a fascist fed, +coldly programming his mighty mainframes to destroy our Bill of Rights. +Sometimes he's a telco bureaucrat, covertly conspiring to register all modems +in the service of an Orwellian surveillance regime. Mostly, though, +this fearsome phantom is a "hacker." He's strange, he doesn't belong, +he's not authorized, he doesn't smell right, he's not keeping his proper place, +he's not one of us. The focus of fear is the hacker, for much the same +reasons that Stanley's fancied assailant is black. + +Stanley's demon can't go away, because he doesn't exist. +Despite singleminded and tremendous effort, he can't be arrested, +sued, jailed, or fired. The only constructive way to do ANYTHING +about him is to learn more about Stanley himself. This learning process +may be repellent, it may be ugly, it may involve grave elements of paranoiac +confusion, but it's necessary. Knowing Stanley requires something more +than class-crossing condescension. It requires more than steely +legal objectivity. It requires human compassion and sympathy. + +To know Stanley is to know his demon. If you know the other guy's demon, +then maybe you'll come to know some of your own. You'll be able to +separate reality from illusion. And then you won't do your cause, +and yourself, more harm than good. Like poor damned Stanley from Chicago did. + +# + +The Federal Computer Investigations Committee (FCIC) is the most important +and influential organization in the realm of American computer-crime. +Since the police of other countries have largely taken their computer-crime +cues from American methods, the FCIC might well be called the most important +computer crime group in the world. + +It is also, by federal standards, an organization of great unorthodoxy. +State and local investigators mix with federal agents. Lawyers, +financial auditors and computer-security programmers trade notes +with street cops. Industry vendors and telco security people show up +to explain their gadgetry and plead for protection and justice. +Private investigators, think-tank experts and industry pundits throw in +their two cents' worth. The FCIC is the antithesis of a formal bureaucracy. + +Members of the FCIC are obscurely proud of this fact; they recognize their +group as aberrant, but are entirely convinced that this, for them, +outright WEIRD behavior is nevertheless ABSOLUTELY NECESSARY +to get their jobs done. + +FCIC regulars --from the Secret Service, the FBI, the IRS, +the Department of Labor, the offices of federal attorneys, +state police, the Air Force, from military intelligence-- +often attend meetings, held hither and thither across the country, +at their own expense. The FCIC doesn't get grants. It doesn't +charge membership fees. It doesn't have a boss. It has no headquarters-- +just a mail drop in Washington DC, at the Fraud Division of the Secret Service. +It doesn't have a budget. It doesn't have schedules. It meets three times +a year--sort of. Sometimes it issues publications, but the FCIC +has no regular publisher, no treasurer, not even a secretary. +There are no minutes of FCIC meetings. Non-federal people are considered +"non-voting members," but there's not much in the way of elections. +There are no badges, lapel pins or certificates of membership. +Everyone is on a first-name basis. There are about forty of them. +Nobody knows how many, exactly. People come, people go-- +sometimes people "go" formally but still hang around anyway. +Nobody has ever exactly figured out what "membership" of this +"Committee" actually entails. + +Strange as this may seem to some, to anyone familiar with the social world +of computing, the "organization" of the FCIC is very recognizable. + +For years now, economists and management theorists have speculated +that the tidal wave of the information revolution would destroy rigid, +pyramidal bureaucracies, where everything is top-down and +centrally controlled. Highly trained "employees" would take on +much greater autonomy, being self-starting, and self-motivating, +moving from place to place, task to task, with great speed and fluidity. +"Ad-hocracy" would rule, with groups of people spontaneously knitting +together across organizational lines, tackling the problem at hand, +applying intense computer-aided expertise to it, and then vanishing +whence they came. + +This is more or less what has actually happened in the world of +federal computer investigation. With the conspicuous exception +of the phone companies, which are after all over a hundred years old, +practically EVERY organization that plays any important role in this book +functions just like the FCIC. The Chicago Task Force, the Arizona +Racketeering Unit, the Legion of Doom, the Phrack crowd, the +Electronic Frontier Foundation--they ALL look and act like "tiger teams" +or "user's groups." They are all electronic ad-hocracies leaping up +spontaneously to attempt to meet a need. + +Some are police. Some are, by strict definition, criminals. +Some are political interest-groups. But every single group +has that same quality of apparent spontaneity--"Hey, gang! +My uncle's got a barn--let's put on a show!" + +Every one of these groups is embarrassed by this "amateurism," +and, for the sake of their public image in a world of non-computer people, +they all attempt to look as stern and formal and impressive as possible. +These electronic frontier-dwellers resemble groups of nineteenth-century +pioneers hankering after the respectability of statehood. +There are however, two crucial differences in the historical experience +of these "pioneers" of the nineteeth and twenty-first centuries. + +First, powerful information technology DOES play into the hands of small, +fluid, loosely organized groups. There have always been "pioneers," +"hobbyists," "amateurs," "dilettantes," "volunteers," "movements," +"users' groups" and "blue-ribbon panels of experts" around. +But a group of this kind--when technically equipped to ship +huge amounts of specialized information, at lightning speed, +to its members, to government, and to the press--is simply +a different kind of animal. It's like the difference between +an eel and an electric eel. + +The second crucial change is that American society is currently +in a state approaching permanent technological revolution. +In the world of computers particularly, it is practically impossible +to EVER stop being a "pioneer," unless you either drop dead or +deliberately jump off the bus. The scene has never slowed down +enough to become well-institutionalized. And after twenty, thirty, +forty years the "computer revolution" continues to spread, +to permeate new corners of society. Anything that really works +is already obsolete. + +If you spend your entire working life as a "pioneer," the word "pioneer" +begins to lose its meaning. Your way of life looks less and less like +an introduction to something else" more stable and organized, +and more and more like JUST THE WAY THINGS ARE. A "permanent revolution" +is really a contradiction in terms. If "turmoil" lasts long enough, +it simply becomes A NEW KIND OF SOCIETY--still the same game of history, +but new players, new rules. + +Apply this to the world of late twentieth-century law enforcement, +and the implications are novel and puzzling indeed. Any bureaucratic +rulebook you write about computer-crime will be flawed when you write it, +and almost an antique by the time it sees print. The fluidity and fast +reactions of the FCIC give them a great advantage in this regard, +which explains their success. Even with the best will in the world +(which it does not, in fact, possess) it is impossible for an organization +the size of the U.S. Federal Bureau of Investigation to get up to speed +on the theory and practice of computer crime. If they tried to train all +their agents to do this, it would be SUICIDAL, as they would NEVER BE ABLE +TO DO ANYTHING ELSE. + +The FBI does try to train its agents in the basics of electronic crime, +at their base in Quantico, Virginia. And the Secret Service, along with +many other law enforcement groups, runs quite successful and well-attended +training courses on wire fraud, business crime, and computer intrusion +at the Federal Law Enforcement Training Center (FLETC, pronounced "fletsy") +in Glynco, Georgia. But the best efforts of these bureaucracies does not +remove the absolute need for a "cutting-edge mess" like the FCIC. + +For you see--the members of FCIC ARE the trainers of the rest +of law enforcement. Practically and literally speaking, +they are the Glynco computer-crime faculty by another name. +If the FCIC went over a cliff on a bus, the U.S. law enforcement +community would be rendered deaf dumb and blind in the world +of computer crime, and would swiftly feel a desperate need +to reinvent them. And this is no time to go starting from scratch. + +On June 11, 1991, I once again arrived in Phoenix, Arizona, +for the latest meeting of the Federal Computer Investigations Committee. +This was more or less the twentieth meeting of this stellar group. +The count was uncertain, since nobody could figure out whether to +include the meetings of "the Colluquy," which is what the FCIC +was called in the mid-1980s before it had even managed to obtain +the dignity of its own acronym. + +Since my last visit to Arizona, in May, the local AzScam bribery scandal +had resolved itself in a general muddle of humiliation. The Phoenix chief +of police, whose agents had videotaped nine state legislators up to no good, +had resigned his office in a tussle with the Phoenix city council over +the propriety of his undercover operations. + +The Phoenix Chief could now join Gail Thackeray and eleven of her closest +associates in the shared experience of politically motivated unemployment. +As of June, resignations were still continuing at the Arizona Attorney +General's office, which could be interpreted as either a New Broom +Sweeping Clean or a Night of the Long Knives Part II, depending on +your point of view. + +The meeting of FCIC was held at the Scottsdale Hilton Resort. +Scottsdale is a wealthy suburb of Phoenix, known as "Scottsdull" +to scoffing local trendies, but well-equipped with posh shopping-malls +and manicured lawns, while conspicuously undersupplied with homeless derelicts. +The Scottsdale Hilton Resort was a sprawling hotel in postmodern +crypto-Southwestern style. It featured a "mission bell tower" +plated in turquoise tile and vaguely resembling a Saudi minaret. + +Inside it was all barbarically striped Santa Fe Style decor. +There was a health spa downstairs and a large oddly-shaped +pool in the patio. A poolside umbrella-stand offered Ben and Jerry's +politically correct Peace Pops. + +I registered as a member of FCIC, attaining a handy discount rate, +then went in search of the Feds. Sure enough, at the back of the +hotel grounds came the unmistakable sound of Gail Thackeray +holding forth. + +Since I had also attended the Computers Freedom and Privacy conference +(about which more later), this was the second time I had seen Thackeray +in a group of her law enforcement colleagues. Once again I was struck +by how simply pleased they seemed to see her. It was natural that she'd +get SOME attention, as Gail was one of two women in a group of some thirty men; +but there was a lot more to it than that. + +Gail Thackeray personifies the social glue of the FCIC. They could give +a damn about her losing her job with the Attorney General. They were sorry +about it, of course, but hell, they'd all lost jobs. If they were the kind +of guys who liked steady boring jobs, they would never have gotten into +computer work in the first place. + +I wandered into her circle and was immediately introduced to five strangers. +The conditions of my visit at FCIC were reviewed. I would not quote +anyone directly. I would not tie opinions expressed to the agencies +of the attendees. I would not (a purely hypothetical example) +report the conversation of a guy from the Secret Service talking +quite civilly to a guy from the FBI, as these two agencies NEVER +talk to each other, and the IRS (also present, also hypothetical) +NEVER TALKS TO ANYBODY. + +Worse yet, I was forbidden to attend the first conference. And I didn't. +I have no idea what the FCIC was up to behind closed doors that afternoon. +I rather suspect that they were engaging in a frank and thorough confession +of their errors, goof-ups and blunders, as this has been a feature of every +FCIC meeting since their legendary Memphis beer-bust of 1986. Perhaps the +single greatest attraction of FCIC is that it is a place where you can go, +let your hair down, and completely level with people who actually comprehend +what you are talking about. Not only do they understand you, but they +REALLY PAY ATTENTION, they are GRATEFUL FOR YOUR INSIGHTS, and they +FORGIVE YOU, which in nine cases out of ten is something even your +boss can't do, because as soon as you start talking "ROM," "BBS," +or "T-1 trunk," his eyes glaze over. + +I had nothing much to do that afternoon. The FCIC were beavering away +in their conference room. Doors were firmly closed, windows too dark +to peer through. I wondered what a real hacker, a computer intruder, +would do at a meeting like this. + +The answer came at once. He would "trash" the place. Not reduce the place +to trash in some orgy of vandalism; that's not the use of the term in the +hacker milieu. No, he would quietly EMPTY THE TRASH BASKETS and silently +raid any valuable data indiscreetly thrown away. + +Journalists have been known to do this. (Journalists hunting information +have been known to do almost every single unethical thing that hackers +have ever done. They also throw in a few awful techniques all their own.) +The legality of `trashing' is somewhat dubious but it is not in fact +flagrantly illegal. It was, however, absurd to contemplate trashing the FCIC. +These people knew all about trashing. I wouldn't last fifteen seconds. + +The idea sounded interesting, though. I'd been hearing a lot about +the practice lately. On the spur of the moment, I decided I would try +trashing the office ACROSS THE HALL from the FCIC, an area which had +nothing to do with the investigators. + +The office was tiny; six chairs, a table. . . . Nevertheless, it was open, +so I dug around in its plastic trash can. + +To my utter astonishment, I came up with the torn scraps of a SPRINT +long-distance phone bill. More digging produced a bank statement +and the scraps of a hand-written letter, along with gum, cigarette ashes, +candy wrappers and a day-old-issue of USA TODAY. + +The trash went back in its receptacle while the scraps of data went into +my travel bag. I detoured through the hotel souvenir shop for some +Scotch tape and went up to my room. + +Coincidence or not, it was quite true. Some poor soul had, in fact, +thrown a SPRINT bill into the hotel's trash. Date May 1991, +total amount due: $252.36. Not a business phone, either, +but a residential bill, in the name of someone called Evelyn +(not her real name). Evelyn's records showed a ## PAST DUE BILL ##! +Here was her nine-digit account ID. Here was a stern computer-printed warning: + +"TREAT YOUR FONCARD AS YOU WOULD ANY CREDIT CARD. TO SECURE AGAINST FRAUD, +NEVER GIVE YOUR FONCARD NUMBER OVER THE PHONE UNLESS YOU INITIATED THE CALL. +IF YOU RECEIVE SUSPICIOUS CALLS PLEASE NOTIFY CUSTOMER SERVICE IMMEDIATELY!" + +I examined my watch. Still plenty of time left for the FCIC to carry on. +I sorted out the scraps of Evelyn's SPRINT bill and re-assembled them with +fresh Scotch tape. Here was her ten-digit FONCARD number. Didn't seem +to have the ID number necessary to cause real fraud trouble. + +I did, however, have Evelyn's home phone number. And the phone numbers +for a whole crowd of Evelyn's long-distance friends and acquaintances. +In San Diego, Folsom, Redondo, Las Vegas, La Jolla, Topeka, and Northampton +Massachusetts. Even somebody in Australia! + +I examined other documents. Here was a bank statement. It was Evelyn's +IRA account down at a bank in San Mateo California (total balance $1877.20). +Here was a charge-card bill for $382.64. She was paying it off bit by bit. + +Driven by motives that were completely unethical and prurient, +I now examined the handwritten notes. They had been torn fairly +thoroughly, so much so that it took me almost an entire five minutes +to reassemble them. + +They were drafts of a love letter. They had been written on +the lined stationery of Evelyn's employer, a biomedical company. +Probably written at work when she should have been doing something else. + +"Dear Bob," (not his real name) "I guess in everyone's life there comes +a time when hard decisions have to be made, and this is a difficult one +for me--very upsetting. Since you haven't called me, and I don't understand +why, I can only surmise it's because you don't want to. I thought I would +have heard from you Friday. I did have a few unusual problems with my phone +and possibly you tried, I hope so. + +"Robert, you asked me to `let go'. . . ." + +The first note ended. UNUSUAL PROBLEMS WITH HER PHONE? +I looked swiftly at the next note. + +"Bob, not hearing from you for the whole weekend has left me very perplexed. . . ." + +Next draft. + +"Dear Bob, there is so much I don't understand right now, and I wish I did. +I wish I could talk to you, but for some unknown reason you have elected not +to call--this is so difficult for me to understand. . . ." + +She tried again. + +"Bob, Since I have always held you in such high esteem, I had every hope that +we could remain good friends, but now one essential ingredient is missing-- +respect. Your ability to discard people when their purpose is served is +appalling to me. The kindest thing you could do for me now is to leave me +alone. You are no longer welcome in my heart or home. . . ." + +Try again. + +"Bob, I wrote a very factual note to you to say how much respect I had lost +for you, by the way you treat people, me in particular, so uncaring and cold. +The kindest thing you can do for me is to leave me alone entirely, +as you are no longer welcome in my heart or home. I would appreciate it +if you could retire your debt to me as soon as possible--I wish no link +to you in any way. Sincerely, Evelyn." + +Good heavens, I thought, the bastard actually owes her money! +I turned to the next page. + +"Bob: very simple. GOODBYE! No more mind games--no more fascination-- +no more coldness--no more respect for you! It's over--Finis. Evie" + +There were two versions of the final brushoff letter, but they read about +the same. Maybe she hadn't sent it. The final item in my illicit and +shameful booty was an envelope addressed to "Bob" at his home address, +but it had no stamp on it and it hadn't been mailed. + +Maybe she'd just been blowing off steam because her rascal boyfriend +had neglected to call her one weekend. Big deal. Maybe they'd kissed +and made up, maybe she and Bob were down at Pop's Chocolate Shop now, +sharing a malted. Sure. + +Easy to find out. All I had to do was call Evelyn up. With a half-clever +story and enough brass-plated gall I could probably trick the truth out of her. +Phone-phreaks and hackers deceive people over the phone all the time. +It's called "social engineering." Social engineering is a very common practice +in the underground, and almost magically effective. Human beings are almost +always the weakest link in computer security. The simplest way to learn +Things You Are Not Meant To Know is simply to call up and exploit the +knowledgeable people. With social engineering, you use the bits of specialized +knowledge you already have as a key, to manipulate people into believing +that you are legitimate. You can then coax, flatter, or frighten them into +revealing almost anything you want to know. Deceiving people (especially +over the phone) is easy and fun. Exploiting their gullibility is very +gratifying; it makes you feel very superior to them. + +If I'd been a malicious hacker on a trashing raid, I would now have Evelyn +very much in my power. Given all this inside data, it wouldn't take much +effort at all to invent a convincing lie. If I were ruthless enough, +and jaded enough, and clever enough, this momentary indiscretion of hers-- +maybe committed in tears, who knows--could cause her a whole world of +confusion and grief. + +I didn't even have to have a MALICIOUS motive. Maybe I'd be "on her side," +and call up Bob instead, and anonymously threaten to break both his kneecaps +if he didn't take Evelyn out for a steak dinner pronto. It was still +profoundly NONE OF MY BUSINESS. To have gotten this knowledge at all +was a sordid act and to use it would be to inflict a sordid injury. + +To do all these awful things would require exactly zero high-tech expertise. +All it would take was the willingness to do it and a certain amount +of bent imagination. + +I went back downstairs. The hard-working FCIC, who had labored forty-five +minutes over their schedule, were through for the day, and adjourned to the +hotel bar. We all had a beer. + +I had a chat with a guy about "Isis," or rather IACIS, +the International Association of Computer Investigation Specialists. +They're into "computer forensics," the techniques of picking computer- +systems apart without destroying vital evidence. IACIS, currently run +out of Oregon, is comprised of investigators in the U.S., Canada, Taiwan +and Ireland. "Taiwan and Ireland?" I said. Are TAIWAN and IRELAND +really in the forefront of this stuff? Well not exactly, my informant +admitted. They just happen to have been the first ones to have caught +on by word of mouth. Still, the international angle counts, because this +is obviously an international problem. Phone-lines go everywhere. + +There was a Mountie here from the Royal Canadian Mounted Police. +He seemed to be having quite a good time. Nobody had flung this +Canadian out because he might pose a foreign security risk. +These are cyberspace cops. They still worry a lot about "jurisdictions," +but mere geography is the least of their troubles. + +NASA had failed to show. NASA suffers a lot from computer intrusions, +in particular from Australian raiders and a well-trumpeted Chaos +Computer Club case, and in 1990 there was a brief press flurry +when it was revealed that one of NASA's Houston branch-exchanges +had been systematically ripped off by a gang of phone-phreaks. +But the NASA guys had had their funding cut. They were stripping everything. + +Air Force OSI, its Office of Special Investigations, is the ONLY federal +entity dedicated full-time to computer security. They'd been expected +to show up in force, but some of them had cancelled--a Pentagon budget pinch. + +As the empties piled up, the guys began joshing around and telling war-stories. +"These are cops," Thackeray said tolerantly. "If they're not talking shop +they talk about women and beer." + +I heard the story about the guy who, asked for "a copy" of a computer disk, +PHOTOCOPIED THE LABEL ON IT. He put the floppy disk onto the glass plate +of a photocopier. The blast of static when the copier worked completely +erased all the real information on the disk. + +Some other poor souls threw a whole bag of confiscated diskettes +into the squad-car trunk next to the police radio. The powerful radio +signal blasted them, too. + +We heard a bit about Dave Geneson, the first computer prosecutor, +a mainframe-runner in Dade County, turned lawyer. Dave Geneson +was one guy who had hit the ground running, a signal virtue +in making the transition to computer-crime. It was generally +agreed that it was easier to learn the world of computers first, +then police or prosecutorial work. You could take certain computer +people and train 'em to successful police work--but of course they +had to have the COP MENTALITY. They had to have street smarts. +Patience. Persistence. And discretion. You've got to make sure +they're not hot-shots, show-offs, "cowboys." + +Most of the folks in the bar had backgrounds in military intelligence, +or drugs, or homicide. It was rudely opined that "military intelligence" +was a contradiction in terms, while even the grisly world of homicide +was considered cleaner than drug enforcement. One guy had been 'way +undercover doing dope-work in Europe for four years straight. +"I'm almost recovered now," he said deadpan, with the acid black humor +that is pure cop. "Hey, now I can say FUCKER without putting MOTHER +in front of it." + +"In the cop world," another guy said earnestly, "everything is good and bad, +black and white. In the computer world everything is gray." + +One guy--a founder of the FCIC, who'd been with the group +since it was just the Colluquy--described his own introduction +to the field. He'd been a Washington DC homicide guy called in +on a "hacker" case. From the word "hacker," he naturally assumed +he was on the trail of a knife-wielding marauder, and went to the +computer center expecting blood and a body. When he finally figured +out what was happening there (after loudly demanding, in vain, +that the programmers "speak English"), he called headquarters +and told them he was clueless about computers. They told him nobody +else knew diddly either, and to get the hell back to work. + +So, he said, he had proceeded by comparisons. By analogy. By metaphor. +"Somebody broke in to your computer, huh?" Breaking and entering; +I can understand that. How'd he get in? "Over the phone-lines." +Harassing phone-calls, I can understand that! What we need here +is a tap and a trace! + +It worked. It was better than nothing. And it worked a lot faster +when he got hold of another cop who'd done something similar. +And then the two of them got another, and another, and pretty soon +the Colluquy was a happening thing. It helped a lot that everybody +seemed to know Carlton Fitzpatrick, the data-processing trainer in Glynco. + +The ice broke big-time in Memphis in '86. The Colluquy had attracted +a bunch of new guys--Secret Service, FBI, military, other feds, heavy guys. +Nobody wanted to tell anybody anything. They suspected that if word got back +to the home office they'd all be fired. They passed an uncomfortably +guarded afternoon. + +The formalities got them nowhere. But after the formal session was over, +the organizers brought in a case of beer. As soon as the participants +knocked it off with the bureaucratic ranks and turf-fighting, everything +changed. "I bared my soul," one veteran reminisced proudly. By nightfall +they were building pyramids of empty beer-cans and doing everything +but composing a team fight song. + +FCIC were not the only computer-crime people around. There was DATTA +(District Attorneys' Technology Theft Association), though they mostly +specialized in chip theft, intellectual property, and black-market cases. +There was HTCIA (High Tech Computer Investigators Association), +also out in Silicon Valley, a year older than FCIC and featuring +brilliant people like Donald Ingraham. There was LEETAC +(Law Enforcement Electronic Technology Assistance Committee) +in Florida, and computer-crime units in Illinois and Maryland +and Texas and Ohio and Colorado and Pennsylvania. But these were +local groups. FCIC were the first to really network nationally +and on a federal level. + +FCIC people live on the phone lines. Not on bulletin board systems-- +they know very well what boards are, and they know that boards aren't secure. +Everyone in the FCIC has a voice-phone bill like you wouldn't believe. +FCIC people have been tight with the telco people for a long time. +Telephone cyberspace is their native habitat. + +FCIC has three basic sub-tribes: the trainers, the security people, +and the investigators. That's why it's called an "Investigations +Committee" with no mention of the term "computer-crime"--the dreaded +"C-word." FCIC, officially, is "an association of agencies rather +than individuals;" unofficially, this field is small enough that +the influence of individuals and individual expertise is paramount. +Attendance is by invitation only, and most everyone in FCIC considers +himself a prophet without honor in his own house. + +Again and again I heard this, with different terms but identical +sentiments. "I'd been sitting in the wilderness talking to myself." +"I was totally isolated." "I was desperate." "FCIC is the best +thing there is about computer crime in America." "FCIC is what +really works." "This is where you hear real people telling you +what's really happening out there, not just lawyers picking nits." +"We taught each other everything we knew." + +The sincerity of these statements convinces me that this is true. +FCIC is the real thing and it is invaluable. It's also very sharply +at odds with the rest of the traditions and power structure +in American law enforcement. There probably hasn't been anything +around as loose and go-getting as the FCIC since the start of the +U.S. Secret Service in the 1860s. FCIC people are living like +twenty-first-century people in a twentieth-century environment, +and while there's a great deal to be said for that, there's also +a great deal to be said against it, and those against it happen +to control the budgets. + +I listened to two FCIC guys from Jersey compare life histories. +One of them had been a biker in a fairly heavy-duty gang in the 1960s. +"Oh, did you know so-and-so?" said the other guy from Jersey. +"Big guy, heavyset?" + +"Yeah, I knew him." + +"Yeah, he was one of ours. He was our plant in the gang." + +"Really? Wow! Yeah, I knew him. Helluva guy." + +Thackeray reminisced at length about being tear-gassed blind +in the November 1969 antiwar protests in Washington Circle, +covering them for her college paper. "Oh yeah, I was there," +said another cop. "Glad to hear that tear gas hit somethin'. +Haw haw haw." He'd been so blind himself, he confessed, +that later that day he'd arrested a small tree. + +FCIC are an odd group, sifted out by coincidence and necessity, +and turned into a new kind of cop. There are a lot of specialized +cops in the world--your bunco guys, your drug guys, your tax guys, +but the only group that matches FCIC for sheer isolation are probably +the child-pornography people. Because they both deal with conspirators +who are desperate to exchange forbidden data and also desperate to hide; +and because nobody else in law enforcement even wants to hear about it. + +FCIC people tend to change jobs a lot. They tend not to get the equipment +and training they want and need. And they tend to get sued quite often. + +As the night wore on and a band set up in the bar, the talk grew darker. +Nothing ever gets done in government, someone opined, until there's +a DISASTER. Computing disasters are awful, but there's no denying +that they greatly help the credibility of FCIC people. The Internet Worm, +for instance. "For years we'd been warning about that--but it's nothing +compared to what's coming." They expect horrors, these people. +They know that nothing will really get done until there is a horror. + +# + +Next day we heard an extensive briefing from a guy who'd been a computer cop, +gotten into hot water with an Arizona city council, and now installed +computer networks for a living (at a considerable rise in pay). +He talked about pulling fiber-optic networks apart. + +Even a single computer, with enough peripherals, is a literal +"network"--a bunch of machines all cabled together, generally +with a complexity that puts stereo units to shame. FCIC people +invent and publicize methods of seizing computers and maintaining +their evidence. Simple things, sometimes, but vital rules of thumb +for street cops, who nowadays often stumble across a busy computer +in the midst of a drug investigation or a white-collar bust. +For instance: Photograph the system before you touch it. +Label the ends of all the cables before you detach anything. +"Park" the heads on the disk drives before you move them. +Get the diskettes. Don't put the diskettes in magnetic fields. +Don't write on diskettes with ballpoint pens. Get the manuals. +Get the printouts. Get the handwritten notes. Copy data before +you look at it, and then examine the copy instead of the original. + +Now our lecturer distributed copied diagrams of a typical LAN +or "Local Area Network", which happened to be out of Connecticut. +ONE HUNDRED AND FIFTY-NINE desktop computers, each with its own +peripherals. Three "file servers." Five "star couplers" +each with thirty-two ports. One sixteen-port coupler +off in the corner office. All these machines talking to each other, +distributing electronic mail, distributing software, distributing, +quite possibly, criminal evidence. All linked by high-capacity +fiber-optic cable. A bad guy--cops talk a about "bad guys" +--might be lurking on PC #47 lot or #123 and distributing +his ill doings onto some dupe's "personal" machine in +another office--or another floor--or, quite possibly, +two or three miles away! Or, conceivably, the evidence might +be "data-striped"--split up into meaningless slivers stored, +one by one, on a whole crowd of different disk drives. + +The lecturer challenged us for solutions. I for one was utterly clueless. +As far as I could figure, the Cossacks were at the gate; there were probably +more disks in this single building than were seized during the entirety +of Operation Sundevil. + +"Inside informant," somebody said. Right. There's always the human angle, +something easy to forget when contemplating the arcane recesses of high +technology. Cops are skilled at getting people to talk, and computer people, +given a chair and some sustained attention, will talk about their computers +till their throats go raw. There's a case on record of a single question-- +"How'd you do it?"--eliciting a forty-five-minute videotaped confession +from a computer criminal who not only completely incriminated himself +but drew helpful diagrams. + +Computer people talk. Hackers BRAG. Phone-phreaks +talk PATHOLOGICALLY--why else are they stealing phone-codes, +if not to natter for ten hours straight to their friends +on an opposite seaboard? Computer-literate people do +in fact possess an arsenal of nifty gadgets and techniques +that would allow them to conceal all kinds of exotic skullduggery, +and if they could only SHUT UP about it, they could probably +get away with all manner of amazing information-crimes. +But that's just not how it works--or at least, +that's not how it's worked SO FAR. + +Most every phone-phreak ever busted has swiftly implicated his mentors, +his disciples, and his friends. Most every white-collar computer-criminal, +smugly convinced that his clever scheme is bulletproof, swiftly learns +otherwise when, for the first time in his life, an actual no-kidding +policeman leans over, grabs the front of his shirt, looks him right +in the eye and says: "All right, ASSHOLE--you and me are going downtown!" +All the hardware in the world will not insulate your nerves from +these actual real-life sensations of terror and guilt. + +Cops know ways to get from point A to point Z without thumbing +through every letter in some smart-ass bad-guy's alphabet. +Cops know how to cut to the chase. Cops know a lot of things +other people don't know. + +Hackers know a lot of things other people don't know, too. +Hackers know, for instance, how to sneak into your computer +through the phone-lines. But cops can show up RIGHT ON YOUR DOORSTEP +and carry off YOU and your computer in separate steel boxes. +A cop interested in hackers can grab them and grill them. +A hacker interested in cops has to depend on hearsay, +underground legends, and what cops are willing to publicly reveal. +And the Secret Service didn't get named "the SECRET Service" +because they blab a lot. + +Some people, our lecturer informed us, were under the mistaken +impression that it was "impossible" to tap a fiber-optic line. +Well, he announced, he and his son had just whipped up a +fiber-optic tap in his workshop at home. He passed it around +the audience, along with a circuit-covered LAN plug-in card +so we'd all recognize one if we saw it on a case. We all had a look. + +The tap was a classic "Goofy Prototype"--a thumb-length rounded +metal cylinder with a pair of plastic brackets on it. +From one end dangled three thin black cables, each of which ended +in a tiny black plastic cap. When you plucked the safety-cap +off the end of a cable, you could see the glass fiber-- +no thicker than a pinhole. + +Our lecturer informed us that the metal cylinder was a +"wavelength division multiplexer." Apparently, what one did +was to cut the fiber-optic cable, insert two of the legs into +the cut to complete the network again, and then read any passing data +on the line by hooking up the third leg to some kind of monitor. +Sounded simple enough. I wondered why nobody had thought of it before. +I also wondered whether this guy's son back at the workshop had any +teenage friends. + +We had a break. The guy sitting next to me was wearing a giveaway +baseball cap advertising the Uzi submachine gun. We had a desultory chat +about the merits of Uzis. Long a favorite of the Secret Service, +it seems Uzis went out of fashion with the advent of the Persian Gulf War, +our Arab allies taking some offense at Americans toting Israeli weapons. +Besides, I was informed by another expert, Uzis jam. The equivalent weapon +of choice today is the Heckler & Koch, manufactured in Germany. + +The guy with the Uzi cap was a forensic photographer. He also did a lot +of photographic surveillance work in computer crime cases. He used to, +that is, until the firings in Phoenix. He was now a private investigator and, +with his wife, ran a photography salon specializing in weddings and portrait +photos. At--one must repeat--a considerable rise in income. + +He was still FCIC. If you were FCIC, and you needed to talk +to an expert about forensic photography, well, there he was, +willing and able. If he hadn't shown up, people would have missed him. + +Our lecturer had raised the point that preliminary investigation +of a computer system is vital before any seizure is undertaken. +It's vital to understand how many machines are in there, what kinds +there are, what kind of operating system they use, how many people +use them, where the actual data itself is stored. To simply barge into +an office demanding "all the computers" is a recipe for swift disaster. + +This entails some discreet inquiries beforehand. In fact, what it +entails is basically undercover work. An intelligence operation. +SPYING, not to put too fine a point on it. + +In a chat after the lecture, I asked an attendee whether "trashing" might work. + +I received a swift briefing on the theory and practice of "trash covers." +Police "trash covers," like "mail covers" or like wiretaps, require the +agreement of a judge. This obtained, the "trashing" work of cops is just +like that of hackers, only more so and much better organized. So much so, +I was informed, that mobsters in Phoenix make extensive use of locked +garbage cans picked up by a specialty high-security trash company. + +In one case, a tiger team of Arizona cops had trashed a local residence +for four months. Every week they showed up on the municipal garbage truck, +disguised as garbagemen, and carried the contents of the suspect cans off +to a shade tree, where they combed through the garbage--a messy task, +especially considering that one of the occupants was undergoing +kidney dialysis. All useful documents were cleaned, dried and examined. +A discarded typewriter-ribbon was an especially valuable source of data, +as its long one-strike ribbon of film contained the contents of every +letter mailed out of the house. The letters were neatly retyped by +a police secretary equipped with a large desk-mounted magnifying glass. + +There is something weirdly disquieting about the whole subject of +"trashing"-- an unsuspected and indeed rather disgusting mode of +deep personal vulnerability. Things that we pass by every day, +that we take utterly for granted, can be exploited with so little work. +Once discovered, the knowledge of these vulnerabilities tend to spread. + +Take the lowly subject of MANHOLE COVERS. The humble manhole cover +reproduces many of the dilemmas of computer-security in miniature. +Manhole covers are, of course, technological artifacts, access-points +to our buried urban infrastructure. To the vast majority of us, +manhole covers are invisible. They are also vulnerable. For many years now, +the Secret Service has made a point of caulking manhole covers along all routes +of the Presidential motorcade. This is, of course, to deter terrorists from +leaping out of underground ambush or, more likely, planting remote-control +car-smashing bombs beneath the street. + +Lately, manhole covers have seen more and more criminal exploitation, +especially in New York City. Recently, a telco in New York City +discovered that a cable television service had been sneaking into +telco manholes and installing cable service alongside the phone-lines-- +WITHOUT PAYING ROYALTIES. New York companies have also suffered a +general plague of (a) underground copper cable theft; (b) dumping of garbage, +including toxic waste, and (c) hasty dumping of murder victims. + +Industry complaints reached the ears of an innovative New England +industrial-security company, and the result was a new product known +as "the Intimidator," a thick titanium-steel bolt with a precisely machined +head that requires a special device to unscrew. All these "keys" have registered +serial numbers kept on file with the manufacturer. There are now some +thousands of these "Intimidator" bolts being sunk into American pavements +wherever our President passes, like some macabre parody of strewn roses. +They are also spreading as fast as steel dandelions around US military bases +and many centers of private industry. + +Quite likely it has never occurred to you to peer under a manhole cover, +perhaps climb down and walk around down there with a flashlight, just to see +what it's like. Formally speaking, this might be trespassing, but if you +didn't hurt anything, and didn't make an absolute habit of it, nobody would +really care. The freedom to sneak under manholes was likely a freedom +you never intended to exercise. + +You now are rather less likely to have that freedom at all. +You may never even have missed it until you read about it here, +but if you're in New York City it's gone, and elsewhere it's likely going. +This is one of the things that crime, and the reaction to +crime, does to us. + +The tenor of the meeting now changed as the Electronic Frontier Foundation +arrived. The EFF, whose personnel and history will be examined in detail +in the next chapter, are a pioneering civil liberties group who arose in +direct response to the Hacker Crackdown of 1990. + +Now Mitchell Kapor, the Foundation's president, and Michael Godwin, +its chief attorney, were confronting federal law enforcement MANO A MANO +for the first time ever. Ever alert to the manifold uses of publicity, +Mitch Kapor and Mike Godwin had brought their own journalist in tow: +Robert Draper, from Austin, whose recent well-received book about +ROLLING STONE magazine was still on the stands. Draper was on assignment +for TEXAS MONTHLY. + +The Steve Jackson/EFF civil lawsuit against the Chicago Computer Fraud +and Abuse Task Force was a matter of considerable regional interest in Texas. +There were now two Austinite journalists here on the case. In fact, +counting Godwin (a former Austinite and former journalist) there were +three of us. Lunch was like Old Home Week. + +Later, I took Draper up to my hotel room. We had a long frank talk +about the case, networking earnestly like a miniature freelance-journo +version of the FCIC: privately confessing the numerous blunders +of journalists covering the story, and trying hard to figure out +who was who and what the hell was really going on out there. +I showed Draper everything I had dug out of the Hilton trashcan. +We pondered the ethics of "trashing" for a while, and agreed +that they were dismal. We also agreed that finding a SPRINT +bill on your first time out was a heck of a coincidence. + +First I'd "trashed"--and now, mere hours later, I'd bragged to someone else. +Having entered the lifestyle of hackerdom, I was now, unsurprisingly, +following its logic. Having discovered something remarkable through +a surreptitious action, I of course HAD to "brag," and to drag the passing +Draper into my iniquities. I felt I needed a witness. Otherwise nobody +would have believed what I'd discovered. . . . + +Back at the meeting, Thackeray cordially, if rather tentatively, +introduced Kapor and Godwin to her colleagues. Papers were distributed. +Kapor took center stage. The brilliant Bostonian high-tech entrepreneur, +normally the hawk in his own administration and quite an effective +public speaker, seemed visibly nervous, and frankly admitted as much. +He began by saying he consided computer-intrusion to be morally wrong, +and that the EFF was not a "hacker defense fund," despite what had appeared +in print. Kapor chatted a bit about the basic motivations of his group, +emphasizing their good faith and willingness to listen and seek common ground +with law enforcement--when, er, possible. + +Then, at Godwin's urging, Kapor suddenly remarked that EFF's own Internet +machine had been "hacked" recently, and that EFF did not consider +this incident amusing. + +After this surprising confession, things began to loosen up +quite rapidly. Soon Kapor was fielding questions, parrying objections, +challenging definitions, and juggling paradigms with something akin +to his usual gusto. + +Kapor seemed to score quite an effect with his shrewd and skeptical analysis +of the merits of telco "Caller-ID" services. (On this topic, FCIC and EFF +have never been at loggerheads, and have no particular established earthworks +to defend.) Caller-ID has generally been promoted as a privacy service +for consumers, a presentation Kapor described as a "smokescreen," +the real point of Caller-ID being to ALLOW CORPORATE CUSTOMERS TO BUILD +EXTENSIVE COMMERCIAL DATABASES ON EVERYBODY WHO PHONES OR FAXES THEM. +Clearly, few people in the room had considered this possibility, +except perhaps for two late-arrivals from US WEST RBOC security, +who chuckled nervously. + +Mike Godwin then made an extensive presentation on +"Civil Liberties Implications of Computer Searches and Seizures." +Now, at last, we were getting to the real nitty-gritty here, +real political horse-trading. The audience listened with close +attention, angry mutters rising occasionally: "He's trying to +teach us our jobs!" "We've been thinking about this for years! +We think about these issues every day!" "If I didn't seize the works, +I'd be sued by the guy's victims!" "I'm violating the law if I leave +ten thousand disks full of illegal PIRATED SOFTWARE and STOLEN CODES!" +"It's our job to make sure people don't trash the Constitution-- +we're the DEFENDERS of the Constitution!" "We seize stuff when +we know it will be forfeited anyway as restitution for the victim!" + +"If it's forfeitable, then don't get a search warrant, get a +forfeiture warrant," Godwin suggested coolly. He further remarked +that most suspects in computer crime don't WANT to see their computers +vanish out the door, headed God knew where, for who knows how long. +They might not mind a search, even an extensive search, but they want +their machines searched on-site. + +"Are they gonna feed us?" somebody asked sourly. + +"How about if you take copies of the data?" Godwin parried. + +"That'll never stand up in court." + +"Okay, you make copies, give THEM the copies, and take the originals." + +Hmmm. + +Godwin championed bulletin-board systems as repositories of First Amendment +protected free speech. He complained that federal computer-crime training +manuals gave boards a bad press, suggesting that they are hotbeds of crime +haunted by pedophiles and crooks, whereas the vast majority of the nation's +thousands of boards are completely innocuous, and nowhere near so +romantically suspicious. + +People who run boards violently resent it when their systems are seized, +and their dozens (or hundreds) of users look on in abject horror. +Their rights of free expression are cut short. Their right to associate +with other people is infringed. And their privacy is violated as their +private electronic mail becomes police property. + +Not a soul spoke up to defend the practice of seizing boards. +The issue passed in chastened silence. Legal principles aside-- +(and those principles cannot be settled without laws passed or +court precedents)--seizing bulletin boards has become public-relations +poison for American computer police. + +And anyway, it's not entirely necessary. If you're a cop, you can get 'most +everything you need from a pirate board, just by using an inside informant. +Plenty of vigilantes--well, CONCERNED CITIZENS--will inform police the moment +they see a pirate board hit their area (and will tell the police all about it, +in such technical detail, actually, that you kinda wish they'd shut up). +They will happily supply police with extensive downloads or printouts. +It's IMPOSSIBLE to keep this fluid electronic information out of the +hands of police. + +Some people in the electronic community become enraged at the prospect +of cops "monitoring" bulletin boards. This does have touchy aspects, +as Secret Service people in particular examine bulletin boards with +some regularity. But to expect electronic police to be deaf dumb +and blind in regard to this particular medium rather flies in the face +of common sense. Police watch television, listen to radio, read newspapers +and magazines; why should the new medium of boards be different? +Cops can exercise the same access to electronic information +as everybody else. As we have seen, quite a few computer +police maintain THEIR OWN bulletin boards, including anti-hacker +"sting" boards, which have generally proven quite effective. + +As a final clincher, their Mountie friends in Canada (and colleagues +in Ireland and Taiwan) don't have First Amendment or American +constitutional restrictions, but they do have phone lines, +and can call any bulletin board in America whenever they please. +The same technological determinants that play into the hands of hackers, +phone phreaks and software pirates can play into the hands of police. +"Technological determinants" don't have ANY human allegiances. +They're not black or white, or Establishment or Underground, +or pro-or-anti anything. + +Godwin complained at length about what he called "the Clever Hobbyist +hypothesis" --the assumption that the "hacker" you're busting is clearly +a technical genius, and must therefore by searched with extreme thoroughness. +So: from the law's point of view, why risk missing anything? Take the works. +Take the guy's computer. Take his books. Take his notebooks. +Take the electronic drafts of his love letters. Take his Walkman. +Take his wife's computer. Take his dad's computer. Take his kid +sister's computer. Take his employer's computer. Take his compact disks-- +they MIGHT be CD-ROM disks, cunningly disguised as pop music. +Take his laser printer--he might have hidden something vital in the +printer's 5meg of memory. Take his software manuals and hardware +documentation. Take his science-fiction novels and his simulation- +gaming books. Take his Nintendo Game-Boy and his Pac-Man arcade game. +Take his answering machine, take his telephone out of the wall. +Take anything remotely suspicious. + +Godwin pointed out that most "hackers" are not, in fact, clever +genius hobbyists. Quite a few are crooks and grifters who don't +have much in the way of technical sophistication; just some rule-of-thumb +rip-off techniques. The same goes for most fifteen-year-olds who've +downloaded a code-scanning program from a pirate board. There's no +real need to seize everything in sight. It doesn't require an entire +computer system and ten thousand disks to prove a case in court. + +What if the computer is the instrumentality of a crime? someone demanded. + +Godwin admitted quietly that the doctrine of seizing the instrumentality +of a crime was pretty well established in the American legal system. + +The meeting broke up. Godwin and Kapor had to leave. Kapor was testifying +next morning before the Massachusetts Department Of Public Utility, +about ISDN narrowband wide-area networking. + +As soon as they were gone, Thackeray seemed elated. +She had taken a great risk with this. Her colleagues had not, +in fact, torn Kapor and Godwin's heads off. She was very proud of them, +and told them so. + +"Did you hear what Godwin said about INSTRUMENTALITY OF A CRIME?" +she exulted, to nobody in particular. "Wow, that means +MITCH ISN'T GOING TO SUE ME." + +# + +America's computer police are an interesting group. +As a social phenomenon they are far more interesting, +and far more important, than teenage phone phreaks +and computer hackers. First, they're older and wiser; +not dizzy hobbyists with leaky morals, but seasoned adult +professionals with all the responsibilities of public service. +And, unlike hackers, they possess not merely TECHNICAL +power alone, but heavy-duty legal and social authority. + +And, very interestingly, they are just as much at +sea in cyberspace as everyone else. They are not +happy about this. Police are authoritarian by nature, +and prefer to obey rules and precedents. (Even those police +who secretly enjoy a fast ride in rough territory will soberly +disclaim any "cowboy" attitude.) But in cyberspace there ARE +no rules and precedents. They are groundbreaking pioneers, +Cyberspace Rangers, whether they like it or not. + +In my opinion, any teenager enthralled by computers, +fascinated by the ins and outs of computer security, +and attracted by the lure of specialized forms of knowledge and power, +would do well to forget all about "hacking" and set his (or her) +sights on becoming a fed. Feds can trump hackers at almost every +single thing hackers do, including gathering intelligence, +undercover disguise, trashing, phone-tapping, building dossiers, +networking, and infiltrating computer systems--CRIMINAL computer systems. +Secret Service agents know more about phreaking, coding and carding +than most phreaks can find out in years, and when it comes to viruses, +break-ins, software bombs and trojan horses, Feds have direct access to red-hot +confidential information that is only vague rumor in the underground. + +And if it's an impressive public rep you're after, there are few people +in the world who can be so chillingly impressive as a well-trained, +well-armed United States Secret Service agent. + +Of course, a few personal sacrifices are necessary in order to obtain +that power and knowledge. First, you'll have the galling discipline +of belonging to a large organization; but the world of computer crime +is still so small, and so amazingly fast-moving, that it will remain +spectacularly fluid for years to come. The second sacrifice is that +you'll have to give up ripping people off. This is not a great loss. +Abstaining from the use of illegal drugs, also necessary, will be a boon +to your health. + +A career in computer security is not a bad choice for a young man +or woman today. The field will almost certainly expand drastically +in years to come. If you are a teenager today, by the time you +become a professional, the pioneers you have read about in this book +will be the grand old men and women of the field, swamped by their many +disciples and successors. Of course, some of them, like William P. Wood +of the 1865 Secret Service, may well be mangled in the whirring machinery +of legal controversy; but by the time you enter the computer-crime field, +it may have stabilized somewhat, while remaining entertainingly challenging. + +But you can't just have a badge. You have to win it. First, there's the +federal law enforcement training. And it's hard--it's a challenge. +A real challenge--not for wimps and rodents. + +Every Secret Service agent must complete gruelling courses at the +Federal Law Enforcement Training Center. (In fact, Secret Service +agents are periodically re-trained during their entire careers.) + +In order to get a glimpse of what this might be like, +I myself travelled to FLETC. + +# + +The Federal Law Enforcement Training Center is a 1500-acre facility +on Georgia's Atlantic coast. It's a milieu of marshgrass, seabirds, +damp, clinging sea-breezes, palmettos, mosquitos, and bats. +Until 1974, it was a Navy Air Base, and still features a working runway, +and some WWII vintage blockhouses and officers' quarters. +The Center has since benefitted by a forty-million-dollar retrofit, +but there's still enough forest and swamp on the facility for the +Border Patrol to put in tracking practice. + +As a town, "Glynco" scarcely exists. The nearest real town is Brunswick, +a few miles down Highway 17, where I stayed at the aptly named Marshview +Holiday Inn. I had Sunday dinner at a seafood restaurant called "Jinright's," +where I feasted on deep-fried alligator tail. This local favorite was +a heaped basket of bite-sized chunks of white, tender, almost fluffy +reptile meat, steaming in a peppered batter crust. Alligator makes +a culinary experience that's hard to forget, especially when liberally +basted with homemade cocktail sauce from a Jinright squeeze-bottle. + +The crowded clientele were tourists, fishermen, local black folks +in their Sunday best, and white Georgian locals who all seemed +to bear an uncanny resemblance to Georgia humorist Lewis Grizzard. + +The 2,400 students from 75 federal agencies who make up the FLETC +population scarcely seem to make a dent in the low-key local scene. +The students look like tourists, and the teachers seem to have taken +on much of the relaxed air of the Deep South. My host was Mr. Carlton +Fitzpatrick, the Program Coordinator of the Financial Fraud Institute. +Carlton Fitzpatrick is a mustached, sinewy, well-tanned Alabama native +somewhere near his late forties, with a fondness for chewing tobacco, +powerful computers, and salty, down-home homilies. We'd met before, +at FCIC in Arizona. + +The Financial Fraud Institute is one of the nine divisions at FLETC. +Besides Financial Fraud, there's Driver & Marine, Firearms, +and Physical Training. These are specialized pursuits. +There are also five general training divisions: Basic Training, +Operations, Enforcement Techniques, Legal Division, and Behavioral Science. + +Somewhere in this curriculum is everything necessary to turn green college +graduates into federal agents. First they're given ID cards. Then they get +the rather miserable-looking blue coveralls known as "smurf suits." +The trainees are assigned a barracks and a cafeteria, and immediately +set on FLETC's bone-grinding physical training routine. Besides the +obligatory daily jogging--(the trainers run up danger flags beside +the track when the humidity rises high enough to threaten heat stroke)-- +here's the Nautilus machines, the martial arts, the survival skills. . . . + +The eighteen federal agencies who maintain on-site academies at FLETC +employ a wide variety of specialized law enforcement units, some of them +rather arcane. There's Border Patrol, IRS Criminal Investigation Division, +Park Service, Fish and Wildlife, Customs, Immigration, Secret Service and +the Treasury's uniformed subdivisions. . . . If you're a federal cop +and you don't work for the FBI, you train at FLETC. This includes people +as apparently obscure as the agents of the Railroad Retirement Board +Inspector General. Or the Tennessee Valley Authority Police, +who are in fact federal police officers, and can and do arrest criminals +on the federal property of the Tennessee Valley Authority. + +And then there are the computer-crime people. All sorts, all backgrounds. +Mr. Fitzpatrick is not jealous of his specialized knowledge. Cops all over, +in every branch of service, may feel a need to learn what he can teach. +Backgrounds don't matter much. Fitzpatrick himself was originally a +Border Patrol veteran, then became a Border Patrol instructor at FLETC. +His Spanish is still fluent--but he found himself strangely fascinated +when the first computers showed up at the Training Center. Fitzpatrick +did have a background in electrical engineering, and though he never +considered himself a computer hacker, he somehow found himself writing +useful little programs for this new and promising gizmo. + +He began looking into the general subject of computers and crime, +reading Donn Parker's books and articles, keeping an ear cocked +for war stories, useful insights from the field, the up-and-coming +people of the local computer-crime and high-technology units. . . . +Soon he got a reputation around FLETC as the resident "computer expert," +and that reputation alone brought him more exposure, more experience-- +until one day he looked around, and sure enough he WAS a federal +computer-crime expert. + +In fact, this unassuming, genial man may be THE federal computer-crime expert. +There are plenty of very good computer people, and plenty of very good +federal investigators, but the area where these worlds of expertise overlap +is very slim. And Carlton Fitzpatrick has been right at the center of that +since 1985, the first year of the Colluquy, a group which owes much to +his influence. + +He seems quite at home in his modest, acoustic-tiled office, +with its Ansel Adams-style Western photographic art, a gold-framed +Senior Instructor Certificate, and a towering bookcase crammed with +three-ring binders with ominous titles such as Datapro Reports on +Information Security and CFCA Telecom Security '90. + +The phone rings every ten minutes; colleagues show up at the door +to chat about new developments in locksmithing or to shake their heads +over the latest dismal developments in the BCCI global banking scandal. + +Carlton Fitzpatrick is a fount of computer-crime war-stories, +related in an acerbic drawl. He tells me the colorful tale +of a hacker caught in California some years back. He'd been +raiding systems, typing code without a detectable break, +for twenty, twenty-four, thirty-six hours straight. Not just +logged on--TYPING. Investigators were baffled. Nobody +could do that. Didn't he have to go to the bathroom? +Was it some kind of automatic keyboard-whacking device +that could actually type code? + +A raid on the suspect's home revealed a situation of astonishing squalor. +The hacker turned out to be a Pakistani computer-science student who had +flunked out of a California university. He'd gone completely underground +as an illegal electronic immigrant, and was selling stolen phone-service +to stay alive. The place was not merely messy and dirty, but in a state +of psychotic disorder. Powered by some weird mix of culture shock, +computer addiction, and amphetamines, the suspect had in fact been sitting +in front of his computer for a day and a half straight, with snacks and +drugs at hand on the edge of his desk and a chamber-pot under his chair. + +Word about stuff like this gets around in the hacker-tracker community. + +Carlton Fitzpatrick takes me for a guided tour by car around the +FLETC grounds. One of our first sights is the biggest indoor +firing range in the world. There are federal trainees in there, +Fitzpatrick assures me politely, blasting away with a wide variety +of automatic weapons: Uzis, Glocks, AK-47s. . . . He's willing to +take me inside. I tell him I'm sure that's really interesting, +but I'd rather see his computers. Carlton Fitzpatrick seems quite +surprised and pleased. I'm apparently the first journalist he's ever +seen who has turned down the shooting gallery in favor of microchips. + +Our next stop is a favorite with touring Congressmen: the three-mile +long FLETC driving range. Here trainees of the Driver & Marine Division +are taught high-speed pursuit skills, setting and breaking road-blocks, +diplomatic security driving for VIP limousines. . . . A favorite FLETC +pastime is to strap a passing Senator into the passenger seat beside a +Driver & Marine trainer, hit a hundred miles an hour, then take it right into +"the skid-pan," a section of greased track where two tons of Detroit iron +can whip and spin like a hockey puck. + +Cars don't fare well at FLETC. First they're rifled again and again +for search practice. Then they do 25,000 miles of high-speed +pursuit training; they get about seventy miles per set +of steel-belted radials. Then it's off to the skid pan, +where sometimes they roll and tumble headlong in the grease. +When they're sufficiently grease-stained, dented, and creaky, +they're sent to the roadblock unit, where they're battered without pity. +And finally then they're sacrificed to the Bureau of Alcohol, +Tobacco and Firearms, whose trainees learn the ins and outs +of car-bomb work by blowing them into smoking wreckage. + +There's a railroad box-car on the FLETC grounds, and a large +grounded boat, and a propless plane; all training-grounds for searches. +The plane sits forlornly on a patch of weedy tarmac next to an eerie +blockhouse known as the "ninja compound," where anti-terrorism specialists +practice hostage rescues. As I gaze on this creepy paragon of modern +low-intensity warfare, my nerves are jangled by a sudden staccato outburst +of automatic weapons fire, somewhere in the woods to my right. +"Nine-millimeter," Fitzpatrick judges calmly. + +Even the eldritch ninja compound pales somewhat compared +to the truly surreal area known as "the raid-houses." +This is a street lined on both sides with nondescript +concrete-block houses with flat pebbled roofs. +They were once officers' quarters. Now they are training grounds. +The first one to our left, Fitzpatrick tells me, has been specially +adapted for computer search-and-seizure practice. Inside it has been +wired for video from top to bottom, with eighteen pan-and-tilt +remotely controlled videocams mounted on walls and in corners. +Every movement of the trainee agent is recorded live by teachers, +for later taped analysis. Wasted movements, hesitations, possibly lethal +tactical mistakes--all are gone over in detail. + +Perhaps the weirdest single aspect of this building is its front door, +scarred and scuffed all along the bottom, from the repeated impact, +day after day, of federal shoe-leather. + +Down at the far end of the row of raid-houses some people are practicing +a murder. We drive by slowly as some very young and rather nervous-looking +federal trainees interview a heavyset bald man on the raid-house lawn. +Dealing with murder takes a lot of practice; first you have to learn +to control your own instinctive disgust and panic, then you have to learn +to control the reactions of a nerve-shredded crowd of civilians, +some of whom may have just lost a loved one, some of whom may be murderers-- +quite possibly both at once. + +A dummy plays the corpse. The roles of the bereaved, the morbidly curious, +and the homicidal are played, for pay, by local Georgians: waitresses, +musicians, most anybody who needs to moonlight and can learn a script. +These people, some of whom are FLETC regulars year after year, +must surely have one of the strangest jobs in the world. + +Something about the scene: "normal" people in a weird situation, +standing around talking in bright Georgia sunshine, unsuccessfully +pretending that something dreadful has gone on, while a dummy lies +inside on faked bloodstains. . . . While behind this weird masquerade, +like a nested set of Russian dolls, are grim future realities of real death, +real violence, real murders of real people, that these young agents +will really investigate, many times during their careers. . . . +Over and over. . . . Will those anticipated murders look like this, +feel like this--not as "real" as these amateur actors are trying to +make it seem, but both as "real," and as numbingly unreal, as watching +fake people standing around on a fake lawn? Something about this scene +unhinges me. It seems nightmarish to me, Kafkaesque. I simply don't +know how to take it; my head is turned around; I don't know whether to laugh, +cry, or just shudder. + +When the tour is over, Carlton Fitzpatrick and I talk about computers. +For the first time cyberspace seems like quite a comfortable place. +It seems very real to me suddenly, a place where I know what I'm talking about, +a place I'm used to. It's real. "Real." Whatever. + +Carlton Fitzpatrick is the only person I've met in cyberspace circles +who is happy with his present equipment. He's got a 5 Meg RAM PC with +a 112 meg hard disk; a 660 meg's on the way. He's got a Compaq 386 desktop, +and a Zenith 386 laptop with 120 meg. Down the hall is a NEC Multi-Sync 2A +with a CD-ROM drive and a 9600 baud modem with four com-lines. +There's a training minicomputer, and a 10-meg local mini just for the Center, +and a lab-full of student PC clones and half-a-dozen Macs or so. +There's a Data General MV 2500 with 8 meg on board and a 370 meg disk. + +Fitzpatrick plans to run a UNIX board on the Data General when he's +finished beta-testing the software for it, which he wrote himself. +It'll have E-mail features, massive files on all manner of computer-crime +and investigation procedures, and will follow the computer-security +specifics of the Department of Defense "Orange Book." He thinks +it will be the biggest BBS in the federal government. + +Will it have Phrack on it? I ask wryly. + +Sure, he tells me. Phrack, TAP, Computer Underground Digest, +all that stuff. With proper disclaimers, of course. + +I ask him if he plans to be the sysop. Running a system that size is very +time-consuming, and Fitzpatrick teaches two three-hour courses every day. + +No, he says seriously, FLETC has to get its money worth out of the instructors. +He thinks he can get a local volunteer to do it, a high-school student. + +He says a bit more, something I think about an Eagle Scout law-enforcement +liaison program, but my mind has rocketed off in disbelief. + +"You're going to put a TEENAGER in charge of a federal security BBS?" +I'm speechless. It hasn't escaped my notice that the FLETC Financial +Fraud Institute is the ULTIMATE hacker-trashing target; there is stuff in here, +stuff of such utter and consummate cool by every standard of the +digital underground. . . . + +I imagine the hackers of my acquaintance, fainting dead-away from +forbidden-knowledge greed-fits, at the mere prospect of cracking +the superultra top-secret computers used to train the Secret Service +in computer-crime. . . . + +"Uhm, Carlton," I babble, "I'm sure he's a really nice kid and all, +but that's a terrible temptation to set in front of somebody who's, +you know, into computers and just starting out. . . ." + +"Yeah," he says, "that did occur to me." For the first time I begin +to suspect that he's pulling my leg. + +He seems proudest when he shows me an ongoing project called JICC, +Joint Intelligence Control Council. It's based on the services provided +by EPIC, the El Paso Intelligence Center, which supplies data and intelligence +to the Drug Enforcement Administration, the Customs Service, the Coast Guard, +and the state police of the four southern border states. Certain EPIC files +can now be accessed by drug-enforcement police of Central America, +South America and the Caribbean, who can also trade information +among themselves. Using a telecom program called "White Hat," +written by two brothers named Lopez from the Dominican Republic, +police can now network internationally on inexpensive PCs. +Carlton Fitzpatrick is teaching a class of drug-war agents +from the Third World, and he's very proud of their progress. +Perhaps soon the sophisticated smuggling networks of the +Medellin Cartel will be matched by a sophisticated computer +network of the Medellin Cartel's sworn enemies. They'll track boats, +track contraband, track the international drug-lords who now leap over +borders with great ease, defeating the police through the clever use +of fragmented national jurisdictions. + +JICC and EPIC must remain beyond the scope of this book. +They seem to me to be very large topics fraught with complications +that I am not fit to judge. I do know, however, that the international, +computer-assisted networking of police, across national boundaries, +is something that Carlton Fitzpatrick considers very important, +a harbinger of a desirable future. I also know that networks +by their nature ignore physical boundaries. And I also know +that where you put communications you put a community, +and that when those communities become self-aware +they will fight to preserve themselves and to expand their influence. +I make no judgements whether this is good or bad. +It's just cyberspace; it's just the way things are. + +I asked Carlton Fitzpatrick what advice he would have for +a twenty-year-old who wanted to shine someday in the world +of electronic law enforcement. + +He told me that the number one rule was simply not to be +scared of computers. You don't need to be an obsessive +"computer weenie," but you mustn't be buffaloed just because +some machine looks fancy. The advantages computers give +smart crooks are matched by the advantages they give smart cops. +Cops in the future will have to enforce the law "with their heads, +not their holsters." Today you can make good cases without ever +leaving your office. In the future, cops who resist the computer +revolution will never get far beyond walking a beat. + +I asked Carlton Fitzpatrick if he had some single message for the public; +some single thing that he would most like the American public to know +about his work. + +He thought about it while. "Yes," he said finally. "TELL me the rules, +and I'll TEACH those rules!" He looked me straight in the eye. +"I do the best that I can." + + + +PART FOUR: THE CIVIL LIBERTARIANS + + +The story of the Hacker Crackdown, as we have followed it thus far, +has been technological, subcultural, criminal and legal. +The story of the Civil Libertarians, though it partakes +of all those other aspects, is profoundly and thoroughly POLITICAL. + +In 1990, the obscure, long-simmering struggle over the ownership +and nature of cyberspace became loudly and irretrievably public. +People from some of the oddest corners of American society suddenly +found themselves public figures. Some of these people found this +situation much more than they had ever bargained for. They backpedalled, +and tried to retreat back to the mandarin obscurity of their cozy +subcultural niches. This was generally to prove a mistake. + +But the civil libertarians seized the day in 1990. They found themselves +organizing, propagandizing, podium-pounding, persuading, touring, +negotiating, posing for publicity photos, submitting to interviews, +squinting in the limelight as they tried a tentative, but growingly +sophisticated, buck-and-wing upon the public stage. + +It's not hard to see why the civil libertarians should have +this competitive advantage. + +The hackers of the digital underground are an hermetic elite. +They find it hard to make any remotely convincing case for +their actions in front of the general public. Actually, +hackers roundly despise the "ignorant" public, and have never +trusted the judgement of "the system." Hackers do propagandize, +but only among themselves, mostly in giddy, badly spelled manifestos +of class warfare, youth rebellion or naive techie utopianism. +Hackers must strut and boast in order to establish and preserve +their underground reputations. But if they speak out too loudly +and publicly, they will break the fragile surface-tension of the underground, +and they will be harrassed or arrested. Over the longer term, +most hackers stumble, get busted, get betrayed, or simply give up. +As a political force, the digital underground is hamstrung. + +The telcos, for their part, are an ivory tower under protracted seige. +They have plenty of money with which to push their calculated public image, +but they waste much energy and goodwill attacking one another with +slanderous and demeaning ad campaigns. The telcos have suffered +at the hands of politicians, and, like hackers, they don't trust +the public's judgement. And this distrust may be well-founded. +Should the general public of the high-tech 1990s come to understand +its own best interests in telecommunications, that might well pose +a grave threat to the specialized technical power and authority +that the telcos have relished for over a century. The telcos do +have strong advantages: loyal employees, specialized expertise, +influence in the halls of power, tactical allies in law enforcement, +and unbelievably vast amounts of money. But politically speaking, they lack +genuine grassroots support; they simply don't seem to have many friends. + +Cops know a lot of things other people don't know. +But cops willingly reveal only those aspects of their +knowledge that they feel will meet their institutional +purposes and further public order. Cops have respect, +they have responsibilities, they have power in the streets +and even power in the home, but cops don't do particularly +well in limelight. When pressed, they will step out in the +public gaze to threaten bad-guys, or to cajole prominent citizens, +or perhaps to sternly lecture the naive and misguided. +But then they go back within their time-honored fortress +of the station-house, the courtroom and the rule-book. + +The electronic civil libertarians, however, have proven to be +born political animals. They seemed to grasp very early on +the postmodern truism that communication is power. Publicity is power. +Soundbites are power. The ability to shove one's issue onto the public +agenda--and KEEP IT THERE--is power. Fame is power. Simple personal +fluency and eloquence can be power, if you can somehow catch the +public's eye and ear. + +The civil libertarians had no monopoly on "technical power"-- +though they all owned computers, most were not particularly +advanced computer experts. They had a good deal of money, +but nowhere near the earthshaking wealth and the galaxy +of resources possessed by telcos or federal agencies. +They had no ability to arrest people. They carried +out no phreak and hacker covert dirty-tricks. + +But they really knew how to network. + +Unlike the other groups in this book, the civil libertarians +have operated very much in the open, more or less right +in the public hurly-burly. They have lectured audiences galore +and talked to countless journalists, and have learned to +refine their spiels. They've kept the cameras clicking, +kept those faxes humming, swapped that email, +run those photocopiers on overtime, licked envelopes +and spent small fortunes on airfare and long-distance. +In an information society, this open, overt, obvious activity +has proven to be a profound advantage. + +In 1990, the civil libertarians of cyberspace assembled +out of nowhere in particular, at warp speed. This "group" +(actually, a networking gaggle of interested parties +which scarcely deserves even that loose term) has almost nothing +in the way of formal organization. Those formal civil libertarian +organizations which did take an interest in cyberspace issues, +mainly the Computer Professionals for Social Responsibility +and the American Civil Liberties Union, were carried along +by events in 1990, and acted mostly as adjuncts, +underwriters or launching-pads. + +The civil libertarians nevertheless enjoyed the greatest success +of any of the groups in the Crackdown of 1990. At this writing, +their future looks rosy and the political initiative is firmly in their hands. +This should be kept in mind as we study the highly unlikely lives +and lifestyles of the people who actually made this happen. + +# + +In June 1989, Apple Computer, Inc., of Cupertino, +California, had a problem. Someone had illicitly copied +a small piece of Apple's proprietary software, software +which controlled an internal chip driving the Macintosh +screen display. This Color QuickDraw source code was +a closely guarded piece of Apple's intellectual property. +Only trusted Apple insiders were supposed to possess it. + +But the "NuPrometheus League" wanted things otherwise. +This person (or persons) made several illicit copies +of this source code, perhaps as many as two dozen. +He (or she, or they) then put those illicit floppy disks +into envelopes and mailed them to people all over America: +people in the computer industry who were associated with, +but not directly employed by, Apple Computer. + +The NuPrometheus caper was a complex, highly ideological, +and very hacker-like crime. Prometheus, it will be recalled, +stole the fire of the Gods and gave this potent gift to the +general ranks of downtrodden mankind. A similar god-in-the-manger +attitude was implied for the corporate elite of Apple Computer, +while the "Nu" Prometheus had himself cast in the role of rebel demigod. +The illicitly copied data was given away for free. + +The new Prometheus, whoever he was, escaped the +fate of the ancient Greek Prometheus, who was chained +to a rock for centuries by the vengeful gods while an eagle +tore and ate his liver. On the other hand, NuPrometheus +chickened out somewhat by comparison with his role model. +The small chunk of Color QuickDraw code he had filched +and replicated was more or less useless to Apple's +industrial rivals (or, in fact, to anyone else). +Instead of giving fire to mankind, it was more as if +NuPrometheus had photocopied the schematics for part of a Bic lighter. +The act was not a genuine work of industrial espionage. +It was best interpreted as a symbolic, deliberate slap +in the face for the Apple corporate heirarchy. + +Apple's internal struggles were well-known in the industry. Apple's founders, +Jobs and Wozniak, had both taken their leave long since. Their raucous core +of senior employees had been a barnstorming crew of 1960s Californians, +many of them markedly less than happy with the new button-down multimillion +dollar regime at Apple. Many of the programmers and developers who had +invented the Macintosh model in the early 1980s had also taken their leave of +the company. It was they, not the current masters of Apple's corporate fate, +who had invented the stolen Color QuickDraw code. The NuPrometheus stunt +was well-calculated to wound company morale. + +Apple called the FBI. The Bureau takes an interest in high-profile +intellectual-property theft cases, industrial espionage and theft +of trade secrets. These were likely the right people to call, +and rumor has it that the entities responsible were in fact discovered +by the FBI, and then quietly squelched by Apple management. NuPrometheus +was never publicly charged with a crime, or prosecuted, or jailed. +But there were no further illicit releases of Macintosh internal software. +Eventually the painful issue of NuPrometheus was allowed to fade. + +In the meantime, however, a large number of puzzled bystanders +found themselves entertaining surprise guests from the FBI. + +One of these people was John Perry Barlow. Barlow is a most unusual man, +difficult to describe in conventional terms. He is perhaps best known as +a songwriter for the Grateful Dead, for he composed lyrics for +"Hell in a Bucket," "Picasso Moon," "Mexicali Blues," "I Need a Miracle," +and many more; he has been writing for the band since 1970. + +Before we tackle the vexing question as to why a rock lyricist +should be interviewed by the FBI in a computer-crime case, +it might be well to say a word or two about the Grateful Dead. +The Grateful Dead are perhaps the most successful and long-lasting +of the numerous cultural emanations from the Haight-Ashbury district +of San Francisco, in the glory days of Movement politics and +lysergic transcendance. The Grateful Dead are a nexus, a veritable +whirlwind, of applique decals, psychedelic vans, tie-dyed T-shirts, +earth-color denim, frenzied dancing and open and unashamed drug use. +The symbols, and the realities, of Californian freak power surround +the Grateful Dead like knotted macrame. + +The Grateful Dead and their thousands of Deadhead devotees +are radical Bohemians. This much is widely understood. +Exactly what this implies in the 1990s is rather more problematic. + +The Grateful Dead are among the world's most popular +and wealthy entertainers: number 20, according to Forbes magazine, +right between M.C. Hammer and Sean Connery. In 1990, this jeans-clad +group of purported raffish outcasts earned seventeen million dollars. +They have been earning sums much along this line for quite some time now. + +And while the Dead are not investment bankers or three-piece-suit +tax specialists--they are, in point of fact, hippie musicians-- +this money has not been squandered in senseless Bohemian excess. +The Dead have been quietly active for many years, funding various +worthy activities in their extensive and widespread cultural community. + +The Grateful Dead are not conventional players in the American +power establishment. They nevertheless are something of a force +to be reckoned with. They have a lot of money and a lot of friends +in many places, both likely and unlikely. + +The Dead may be known for back-to-the-earth environmentalist rhetoric, +but this hardly makes them anti-technological Luddites. On the contrary, +like most rock musicians, the Grateful Dead have spent their entire adult +lives in the company of complex electronic equipment. They have funds to burn +on any sophisticated tool and toy that might happen to catch their fancy. +And their fancy is quite extensive. + +The Deadhead community boasts any number of recording engineers, +lighting experts, rock video mavens, electronic technicians +of all descriptions. And the drift goes both ways. Steve Wozniak, +Apple's co-founder, used to throw rock festivals. Silicon Valley rocks out. + +These are the 1990s, not the 1960s. Today, for a surprising number of people +all over America, the supposed dividing line between Bohemian and technician +simply no longer exists. People of this sort may have a set of windchimes +and a dog with a knotted kerchief 'round its neck, but they're also quite +likely to own a multimegabyte Macintosh running MIDI synthesizer software +and trippy fractal simulations. These days, even Timothy Leary himself, +prophet of LSD, does virtual-reality computer-graphics demos in +his lecture tours. + +John Perry Barlow is not a member of the Grateful Dead. He is, however, +a ranking Deadhead. + +Barlow describes himself as a "techno-crank." A vague term like +"social activist" might not be far from the mark, either. +But Barlow might be better described as a "poet"--if one keeps in mind +Percy Shelley's archaic definition of poets as "unacknowledged legislators +of the world." + +Barlow once made a stab at acknowledged legislator status. In 1987, +he narrowly missed the Republican nomination for a seat in the +Wyoming State Senate. Barlow is a Wyoming native, the third-generation +scion of a well-to-do cattle-ranching family. He is in his early forties, +married and the father of three daughters. + +Barlow is not much troubled by other people's narrow notions of consistency. +In the late 1980s, this Republican rock lyricist cattle rancher sold his ranch +and became a computer telecommunications devotee. + +The free-spirited Barlow made this transition with ease. He genuinely +enjoyed computers. With a beep of his modem, he leapt from small-town +Pinedale, Wyoming, into electronic contact with a large and lively crowd +of bright, inventive, technological sophisticates from all over the world. +Barlow found the social milieu of computing attractive: its fast-lane pace, +its blue-sky rhetoric, its open-endedness. Barlow began dabbling in +computer journalism, with marked success, as he was a quick study, +and both shrewd and eloquent. He frequently travelled to San Francisco +to network with Deadhead friends. There Barlow made extensive contacts +throughout the Californian computer community, including friendships +among the wilder spirits at Apple. + +In May 1990, Barlow received a visit from a local Wyoming agent of the FBI. +The NuPrometheus case had reached Wyoming. + +Barlow was troubled to find himself under investigation in an +area of his interests once quite free of federal attention. +He had to struggle to explain the very nature of computer-crime +to a headscratching local FBI man who specialized in cattle-rustling. +Barlow, chatting helpfully and demonstrating the wonders of his modem +to the puzzled fed, was alarmed to find all "hackers" generally under +FBI suspicion as an evil influence in the electronic community. +The FBI, in pursuit of a hacker called "NuPrometheus," were tracing +attendees of a suspect group called the Hackers Conference. + +The Hackers Conference, which had been started in 1984, was a +yearly Californian meeting of digital pioneers and enthusiasts. +The hackers of the Hackers Conference had little if anything to do +with the hackers of the digital underground. On the contrary, +the hackers of this conference were mostly well-to-do Californian +high-tech CEOs, consultants, journalists and entrepreneurs. +(This group of hackers were the exact sort of "hackers" +most likely to react with militant fury at any criminal +degradation of the term "hacker.") + +Barlow, though he was not arrested or accused of a crime, +and though his computer had certainly not gone out the door, +was very troubled by this anomaly. He carried the word to the Well. + +Like the Hackers Conference, "the Well" was an emanation of the +Point Foundation. Point Foundation, the inspiration of a wealthy +Californian 60s radical named Stewart Brand, was to be a major +launch-pad of the civil libertarian effort. + +Point Foundation's cultural efforts, like those of their fellow Bay Area +Californians the Grateful Dead, were multifaceted and multitudinous. +Rigid ideological consistency had never been a strong suit of the +Whole Earth Catalog. This Point publication had enjoyed a strong +vogue during the late 60s and early 70s, when it offered hundreds +of practical (and not so practical) tips on communitarian living, +environmentalism, and getting back-to-the-land. The Whole Earth Catalog, +and its sequels, sold two and half million copies and won a +National Book Award. + +With the slow collapse of American radical dissent, the Whole Earth Catalog +had slipped to a more modest corner of the cultural radar; but in its +magazine incarnation, CoEvolution Quarterly, the Point Foundation +continued to offer a magpie potpourri of "access to tools and ideas." + +CoEvolution Quarterly, which started in 1974, was never a widely +popular magazine. Despite periodic outbreaks of millenarian fervor, +CoEvolution Quarterly failed to revolutionize Western civilization +and replace leaden centuries of history with bright new Californian paradigms. +Instead, this propaganda arm of Point Foundation cakewalked a fine line between +impressive brilliance and New Age flakiness. CoEvolution Quarterly carried +no advertising, cost a lot, and came out on cheap newsprint with modest +black-and-white graphics. It was poorly distributed, and spread mostly +by subscription and word of mouth. + +It could not seem to grow beyond 30,000 subscribers. +And yet--it never seemed to shrink much, either. +Year in, year out, decade in, decade out, some strange +demographic minority accreted to support the magazine. +The enthusiastic readership did not seem to have much +in the way of coherent politics or ideals. It was sometimes +hard to understand what held them together (if the often bitter +debate in the letter-columns could be described as "togetherness"). + +But if the magazine did not flourish, it was resilient; it got by. +Then, in 1984, the birth-year of the Macintosh computer, +CoEvolution Quarterly suddenly hit the rapids. Point Foundation +had discovered the computer revolution. Out came the Whole Earth +Software Catalog of 1984, arousing headscratching doubts among +the tie-dyed faithful, and rabid enthusiasm among the nascent +"cyberpunk" milieu, present company included. Point Foundation +started its yearly Hackers Conference, and began to take an +extensive interest in the strange new possibilities of +digital counterculture. CoEvolution Quarterlyfolded its teepee, +replaced by Whole Earth Software Review and eventually by Whole Earth +Review (the magazine's present incarnation, currently under +the editorship of virtual-reality maven Howard Rheingold). + +1985 saw the birth of the "WELL"--the "Whole Earth 'Lectronic Link." +The Well was Point Foundation's bulletin board system. + +As boards went, the Well was an anomaly from the beginning, +and remained one. It was local to San Francisco. +It was huge, with multiple phonelines and enormous files +of commentary. Its complex UNIX-based software might be +most charitably described as "user-opaque." It was run on +a mainframe out of the rambling offices of a non-profit +cultural foundation in Sausalito. And it was crammed with +fans of the Grateful Dead. + +Though the Well was peopled by chattering hipsters of the Bay Area +counterculture, it was by no means a "digital underground" board. +Teenagers were fairly scarce; most Well users (known as "Wellbeings") +were thirty- and forty-something Baby Boomers. They tended to work +in the information industry: hardware, software, telecommunications, +media, entertainment. Librarians, academics, and journalists were +especially common on the Well, attracted by Point Foundation's +open-handed distribution of "tools and ideas." + +There were no anarchy files on the Well, scarcely a +dropped hint about access codes or credit-card theft. +No one used handles. Vicious "flame-wars" were held to +a comparatively civilized rumble. Debates were sometimes sharp, +but no Wellbeing ever claimed that a rival had disconnected his phone, +trashed his house, or posted his credit card numbers. + +The Well grew slowly as the 1980s advanced. It charged a modest sum +for access and storage, and lost money for years--but not enough to hamper +the Point Foundation, which was nonprofit anyway. By 1990, the Well +had about five thousand users. These users wandered about a gigantic +cyberspace smorgasbord of "Conferences", each conference itself consisting +of a welter of "topics," each topic containing dozens, sometimes hundreds +of comments, in a tumbling, multiperson debate that could last for months +or years on end. + + +In 1991, the Well's list of conferences looked like this: + + +CONFERENCES ON THE WELL + +WELL "Screenzine" Digest (g zine) + +Best of the WELL - vintage material - (g best) + +Index listing of new topics in all conferences - (g newtops) + +Business - Education +---------------------- + +Apple Library Users Group(g alug) Agriculture (g agri) +Brainstorming (g brain) Classifieds (g cla) +Computer Journalism (g cj) Consultants (g consult) +Consumers (g cons) Design (g design) +Desktop Publishing (g desk) Disability (g disability) +Education (g ed) Energy (g energy91) +Entrepreneurs (g entre) Homeowners (g home) +Indexing (g indexing) Investments (g invest) +Kids91 (g kids) Legal (g legal) +One Person Business (g one) +Periodical/newsletter (g per) +Telecomm Law (g tcl) The Future (g fut) +Translators (g trans) Travel (g tra) +Work (g work) + +Electronic Frontier Foundation (g eff) +Computers, Freedom & Privacy (g cfp) +Computer Professionals for Social Responsibility (g cpsr) + +Social - Political - Humanities +--------------------------------- + +Aging (g gray) AIDS (g aids) +Amnesty International (g amnesty) Archives (g arc) +Berkeley (g berk) Buddhist (g wonderland) +Christian (g cross) Couples (g couples) +Current Events (g curr) Dreams (g dream) +Drugs (g dru) East Coast (g east) +Emotional Health@@@@ (g private) Erotica (g eros) +Environment (g env) Firearms (g firearms) +First Amendment (g first) Fringes of Reason (g fringes) +Gay (g gay) Gay (Private)# (g gaypriv) +Geography (g geo) German (g german) +Gulf War (g gulf) Hawaii (g aloha) +Health (g heal) History (g hist) +Holistic (g holi) Interview (g inter) +Italian (g ital) Jewish (g jew) +Liberty (g liberty) Mind (g mind) +Miscellaneous (g misc) Men on the WELL@@ (g mow) +Network Integration (g origin) Nonprofits (g non) +North Bay (g north) Northwest (g nw) +Pacific Rim (g pacrim) Parenting (g par) +Peace (g pea) Peninsula (g pen) +Poetry (g poetry) Philosophy (g phi) +Politics (g pol) Psychology (g psy) +Psychotherapy (g therapy) Recovery## (g recovery) +San Francisco (g sanfran) Scams (g scam) +Sexuality (g sex) Singles (g singles) +Southern (g south) Spanish (g spanish) +Spirituality (g spirit) Tibet (g tibet) +Transportation (g transport) True Confessions (g tru) +Unclear (g unclear) WELL Writer's Workshop@@@(g www) +Whole Earth (g we) Women on the WELL@(g wow) +Words (g words) Writers (g wri) + +@@@@Private Conference - mail wooly for entry +@@@Private conference - mail sonia for entry +@@Private conference - mail flash for entry +@ Private conference - mail reva for entry +# Private Conference - mail hudu for entry +## Private Conference - mail dhawk for entry + +Arts - Recreation - Entertainment +----------------------------------- +ArtCom Electronic Net (g acen) +Audio-Videophilia (g aud) +Bicycles (g bike) Bay Area Tonight@@(g bat) +Boating (g wet) Books (g books) +CD's (g cd) Comics (g comics) +Cooking (g cook) Flying (g flying) +Fun (g fun) Games (g games) +Gardening (g gard) Kids (g kids) +Nightowls@ (g owl) Jokes (g jokes) +MIDI (g midi) Movies (g movies) +Motorcycling (g ride) Motoring (g car) +Music (g mus) On Stage (g onstage) +Pets (g pets) Radio (g rad) +Restaurant (g rest) Science Fiction (g sf) +Sports (g spo) Star Trek (g trek) +Television (g tv) Theater (g theater) +Weird (g weird) Zines/Factsheet Five(g f5) +@Open from midnight to 6am +@@Updated daily + +Grateful Dead +------------- +Grateful Dead (g gd) Deadplan@ (g dp) +Deadlit (g deadlit) Feedback (g feedback) +GD Hour (g gdh) Tapes (g tapes) +Tickets (g tix) Tours (g tours) + +@Private conference - mail tnf for entry + +Computers +----------- +AI/Forth/Realtime (g realtime) Amiga (g amiga) +Apple (g app) Computer Books (g cbook) +Art & Graphics (g gra) Hacking (g hack) +HyperCard (g hype) IBM PC (g ibm) +LANs (g lan) Laptop (g lap) +Macintosh (g mac) Mactech (g mactech) +Microtimes (g microx) Muchomedia (g mucho) +NeXt (g next) OS/2 (g os2) +Printers (g print) Programmer's Net (g net) +Siggraph (g siggraph) Software Design (g sdc) +Software/Programming (g software) +Software Support (g ssc) +Unix (g unix) Windows (g windows) +Word Processing (g word) + +Technical - Communications +---------------------------- +Bioinfo (g bioinfo) Info (g boing) +Media (g media) NAPLPS (g naplps) +Netweaver (g netweaver) Networld (g networld) +Packet Radio (g packet) Photography (g pho) +Radio (g rad) Science (g science) +Technical Writers (g tec) Telecommunications(g tele) +Usenet (g usenet) Video (g vid) +Virtual Reality (g vr) + +The WELL Itself +--------------- +Deeper (g deeper) Entry (g ent) +General (g gentech) Help (g help) +Hosts (g hosts) Policy (g policy) +System News (g news) Test (g test) + +The list itself is dazzling, bringing to the untutored eye +a dizzying impression of a bizarre milieu of mountain-climbing +Hawaiian holistic photographers trading true-life confessions +with bisexual word-processing Tibetans. + +But this confusion is more apparent than real. Each of these conferences +was a little cyberspace world in itself, comprising dozens and perhaps +hundreds of sub-topics. Each conference was commonly frequented by +a fairly small, fairly like-minded community of perhaps a few dozen people. +It was humanly impossible to encompass the entire Well (especially since +access to the Well's mainframe computer was billed by the hour). +Most long-time users contented themselves with a few favorite +topical neighborhoods, with the occasional foray elsewhere +for a taste of exotica. But especially important news items, +and hot topical debates, could catch the attention of the entire +Well community. + +Like any community, the Well had its celebrities, and John Perry Barlow, +the silver-tongued and silver-modemed lyricist of the Grateful Dead, +ranked prominently among them. It was here on the Well that Barlow +posted his true-life tale of computer-crime encounter with the FBI. + +The story, as might be expected, created a great stir. The Well was +already primed for hacker controversy. In December 1989, Harper's magazine +had hosted a debate on the Well about the ethics of illicit computer intrusion. +While over forty various computer-mavens took part, Barlow proved a star +in the debate. So did "Acid Phreak" and "Phiber Optik," a pair of young +New York hacker-phreaks whose skills at telco switching-station intrusion +were matched only by their apparently limitless hunger for fame. +The advent of these two boldly swaggering outlaws in the precincts +of the Well created a sensation akin to that of Black Panthers +at a cocktail party for the radically chic. + +Phiber Optik in particular was to seize the day in 1990. +A devotee of the 2600 circle and stalwart of the New York +hackers' group "Masters of Deception," Phiber Optik was +a splendid exemplar of the computer intruder as committed dissident. +The eighteen-year-old Optik, a high-school dropout and part-time +computer repairman, was young, smart, and ruthlessly obsessive, +a sharp-dressing, sharp-talking digital dude who was utterly +and airily contemptuous of anyone's rules but his own. +By late 1991, Phiber Optik had appeared in Harper's, +Esquire, The New York Times, in countless public debates +and conventions, even on a television show hosted by Geraldo Rivera. + +Treated with gingerly respect by Barlow and other Well mavens, +Phiber Optik swiftly became a Well celebrity. Strangely, despite +his thorny attitude and utter single-mindedness, Phiber Optik seemed +to arouse strong protective instincts in most of the people who met him. +He was great copy for journalists, always fearlessly ready to swagger, +and, better yet, to actually DEMONSTRATE some off-the-wall digital stunt. +He was a born media darling. + +Even cops seemed to recognize that there was something peculiarly unworldly +and uncriminal about this particular troublemaker. He was so bold, +so flagrant, so young, and so obviously doomed, that even those +who strongly disapproved of his actions grew anxious for his welfare, +and began to flutter about him as if he were an endangered seal pup. + +In January 24, 1990 (nine days after the Martin Luther King Day Crash), +Phiber Optik, Acid Phreak, and a third NYC scofflaw named Scorpion were +raided by the Secret Service. Their computers went out the door, +along with the usual blizzard of papers, notebooks, compact disks, +answering machines, Sony Walkmans, etc. Both Acid Phreak and +Phiber Optik were accused of having caused the Crash. + +The mills of justice ground slowly. The case eventually fell into +the hands of the New York State Police. Phiber had lost his machinery +in the raid, but there were no charges filed against him for over a year. +His predicament was extensively publicized on the Well, where it caused +much resentment for police tactics. It's one thing to merely hear about +a hacker raided or busted; it's another to see the police attacking someone +you've come to know personally, and who has explained his motives at length. +Through the Harper's debate on the Well, it had become clear to the +Wellbeings that Phiber Optik was not in fact going to "hurt anything." +In their own salad days, many Wellbeings had tasted tear-gas in pitched +street-battles with police. They were inclined to indulgence for +acts of civil disobedience. + +Wellbeings were also startled to learn of the draconian thoroughness +of a typical hacker search-and-seizure. It took no great stretch of +imagination for them to envision themselves suffering much the same treatment. + +As early as January 1990, sentiment on the Well had already begun to sour, +and people had begun to grumble that "hackers" were getting a raw deal +from the ham-handed powers-that-be. The resultant issue of Harper's +magazine posed the question as to whether computer-intrusion was a "crime" +at all. As Barlow put it later: "I've begun to wonder if we wouldn't +also regard spelunkers as desperate criminals if AT&T owned all the caves." + +In February 1991, more than a year after the raid on his home, +Phiber Optik was finally arrested, and was charged with first-degree +Computer Tampering and Computer Trespass, New York state offenses. +He was also charged with a theft-of-service misdemeanor, involving a complex +free-call scam to a 900 number. Phiber Optik pled guilty to the misdemeanor +charge, and was sentenced to 35 hours of community service. + +This passing harassment from the unfathomable world of straight people +seemed to bother Optik himself little if at all. Deprived of his computer +by the January search-and-seizure, he simply bought himself a portable +computer so the cops could no longer monitor the phone where he lived +with his Mom, and he went right on with his depredations, sometimes on +live radio or in front of television cameras. + +The crackdown raid may have done little to dissuade Phiber Optik, +but its galling affect on the Wellbeings was profound. As 1990 rolled on, +the slings and arrows mounted: the Knight Lightning raid, +the Steve Jackson raid, the nation-spanning Operation Sundevil. +The rhetoric of law enforcement made it clear that there was, +in fact, a concerted crackdown on hackers in progress. + +The hackers of the Hackers Conference, the Wellbeings, and their ilk, +did not really mind the occasional public misapprehension of "hacking;" +if anything, this membrane of differentiation from straight society +made the "computer community" feel different, smarter, better. +They had never before been confronted, however, by a concerted +vilification campaign. + +Barlow's central role in the counter-struggle was one of the major +anomalies of 1990. Journalists investigating the controversy +often stumbled over the truth about Barlow, but they commonly +dusted themselves off and hurried on as if nothing had happened. +It was as if it were TOO MUCH TO BELIEVE that a 1960s freak +from the Grateful Dead had taken on a federal law enforcement operation +head-to-head and ACTUALLY SEEMED TO BE WINNING! + +Barlow had no easily detectable power-base for a political struggle +of this kind. He had no formal legal or technical credentials. +Barlow was, however, a computer networker of truly stellar brilliance. +He had a poet's gift of concise, colorful phrasing. He also had a +journalist's shrewdness, an off-the-wall, self-deprecating wit, +and a phenomenal wealth of simple personal charm. + +The kind of influence Barlow possessed is fairly common currency +in literary, artistic, or musical circles. A gifted critic can +wield great artistic influence simply through defining +the temper of the times, by coining the catch-phrases +and the terms of debate that become the common currency of the period. +(And as it happened, Barlow WAS a part-time art critic, +with a special fondness for the Western art of Frederic Remington.) + +Barlow was the first commentator to adopt William Gibson's +striking science-fictional term "cyberspace" as a synonym +for the present-day nexus of computer and telecommunications networks. +Barlow was insistent that cyberspace should be regarded as +a qualitatively new world, a "frontier." According to Barlow, +the world of electronic communications, now made visible through +the computer screen, could no longer be usefully regarded +as just a tangle of high-tech wiring. Instead, it had become +a PLACE, cyberspace, which demanded a new set of metaphors, +a new set of rules and behaviors. The term, as Barlow employed it, +struck a useful chord, and this concept of cyberspace was picked up +by Time, Scientific American, computer police, hackers, and even +Constitutional scholars. "Cyberspace" now seems likely to become +a permanent fixture of the language. + +Barlow was very striking in person: a tall, craggy-faced, bearded, +deep-voiced Wyomingan in a dashing Western ensemble of jeans, jacket, +cowboy boots, a knotted throat-kerchief and an ever-present Grateful Dead +cloisonne lapel pin. + +Armed with a modem, however, Barlow was truly in his element. +Formal hierarchies were not Barlow's strong suit; he rarely missed +a chance to belittle the "large organizations and their drones," +with their uptight, institutional mindset. Barlow was very much +of the free-spirit persuasion, deeply unimpressed by brass-hats +and jacks-in-office. But when it came to the digital grapevine, +Barlow was a cyberspace ad-hocrat par excellence. + +There was not a mighty army of Barlows. There was only one Barlow, +and he was a fairly anomolous individual. However, the situation only +seemed to REQUIRE a single Barlow. In fact, after 1990, many people +must have concluded that a single Barlow was far more than +they'd ever bargained for. + +Barlow's querulous mini-essay about his encounter with the FBI +struck a strong chord on the Well. A number of other free spirits +on the fringes of Apple Computing had come under suspicion, +and they liked it not one whit better than he did. + +One of these was Mitchell Kapor, the co-inventor of the spreadsheet +program "Lotus 1-2-3" and the founder of Lotus Development Corporation. +Kapor had written-off the passing indignity of being fingerprinted +down at his own local Boston FBI headquarters, but Barlow's post +made the full national scope of the FBI's dragnet clear to Kapor. +The issue now had Kapor's full attention. As the Secret Service +swung into anti-hacker operation nationwide in 1990, Kapor watched +every move with deep skepticism and growing alarm. + +As it happened, Kapor had already met Barlow, who had interviewed Kapor +for a California computer journal. Like most people who met Barlow, +Kapor had been very taken with him. Now Kapor took it upon himself +to drop in on Barlow for a heart-to-heart talk about the situation. + +Kapor was a regular on the Well. Kapor had been a devotee of the +Whole Earth Catalogsince the beginning, and treasured a complete run +of the magazine. And Kapor not only had a modem, but a private jet. +In pursuit of the scattered high-tech investments of Kapor Enterprises Inc., +his personal, multi-million dollar holding company, Kapor commonly crossed +state lines with about as much thought as one might give to faxing a letter. + +The Kapor-Barlow council of June 1990, in Pinedale, Wyoming, was the start +of the Electronic Frontier Foundation. Barlow swiftly wrote a manifesto, +"Crime and Puzzlement," which announced his, and Kapor's, intention +to form a political organization to "raise and disburse funds for education, +lobbying, and litigation in the areas relating to digital speech and the +extension of the Constitution into Cyberspace." + +Furthermore, proclaimed the manifesto, the foundation would +"fund, conduct, and support legal efforts to demonstrate +that the Secret Service has exercised prior restraint on publications, +limited free speech, conducted improper seizure of equipment and data, +used undue force, and generally conducted itself in a fashion which +is arbitrary, oppressive, and unconstitutional." + +"Crime and Puzzlement" was distributed far and wide through computer +networking channels, and also printed in the Whole Earth Review. +The sudden declaration of a coherent, politicized counter-strike +from the ranks of hackerdom electrified the community. Steve Wozniak +(perhaps a bit stung by the NuPrometheus scandal) swiftly offered +to match any funds Kapor offered the Foundation. + +John Gilmore, one of the pioneers of Sun Microsystems, immediately offered +his own extensive financial and personal support. Gilmore, an ardent +libertarian, was to prove an eloquent advocate of electronic privacy issues, +especially freedom from governmental and corporate computer-assisted +surveillance of private citizens. + +A second meeting in San Francisco rounded up further allies: +Stewart Brand of the Point Foundation, virtual-reality pioneers +Jaron Lanier and Chuck Blanchard, network entrepreneur and venture +capitalist Nat Goldhaber. At this dinner meeting, the activists settled on +a formal title: the Electronic Frontier Foundation, Incorporated. +Kapor became its president. A new EFF Conference was opened on +the Point Foundation's Well, and the Well was declared +"the home of the Electronic Frontier Foundation." + +Press coverage was immediate and intense. Like their +nineteenth-century spiritual ancestors, Alexander Graham Bell +and Thomas Watson, the high-tech computer entrepreneurs +of the 1970s and 1980s--people such as Wozniak, Jobs, Kapor, +Gates, and H. Ross Perot, who had raised themselves by their bootstraps +to dominate a glittering new industry--had always made very good copy. + +But while the Wellbeings rejoiced, the press in general seemed +nonplussed by the self-declared "civilizers of cyberspace." +EFF's insistence that the war against "hackers" involved grave +Constitutional civil liberties issues seemed somewhat farfetched, +especially since none of EFF's organizers were lawyers +or established politicians. The business press in particular +found it easier to seize on the apparent core of the story-- +that high-tech entrepreneur Mitchell Kapor had established +a "defense fund for hackers." Was EFF a genuinely important +political development--or merely a clique of wealthy eccentrics, +dabbling in matters better left to the proper authorities? +The jury was still out. + +But the stage was now set for open confrontation. +And the first and the most critical battle was the +hacker show-trial of "Knight Lightning." + +# + +It has been my practice throughout this book to refer to hackers +only by their "handles." There is little to gain by giving +the real names of these people, many of whom are juveniles, +many of whom have never been convicted of any crime, and many +of whom had unsuspecting parents who have already suffered enough. + +But the trial of Knight Lightning on July 24-27, 1990, +made this particular "hacker" a nationally known public figure. +It can do no particular harm to himself or his family if I repeat +the long-established fact that his name is Craig Neidorf (pronounced NYE-dorf). + +Neidorf's jury trial took place in the United States District Court, +Northern District of Illinois, Eastern Division, with the +Honorable Nicholas J. Bua presiding. The United States of America +was the plaintiff, the defendant Mr. Neidorf. The defendant's attorney +was Sheldon T. Zenner of the Chicago firm of Katten, Muchin and Zavis. + +The prosecution was led by the stalwarts of the Chicago Computer Fraud +and Abuse Task Force: William J. Cook, Colleen D. Coughlin, and +David A. Glockner, all Assistant United States Attorneys. +The Secret Service Case Agent was Timothy M. Foley. + +It will be recalled that Neidorf was the co-editor of an underground hacker +"magazine" called Phrack. Phrack was an entirely electronic publication, +distributed through bulletin boards and over electronic networks. +It was amateur publication given away for free. Neidorf had never made +any money for his work in Phrack. Neither had his unindicted co-editor +"Taran King" or any of the numerous Phrack contributors. + +The Chicago Computer Fraud and Abuse Task Force, however, +had decided to prosecute Neidorf as a fraudster. +To formally admit that Phrack was a "magazine" +and Neidorf a "publisher" was to open a prosecutorial +Pandora's Box of First Amendment issues. To do this +was to play into the hands of Zenner and his EFF advisers, +which now included a phalanx of prominent New York civil rights +lawyers as well as the formidable legal staff of Katten, Muchin and Zavis. +Instead, the prosecution relied heavily on the issue of access device fraud: +Section 1029 of Title 18, the section from which the Secret Service drew +its most direct jurisdiction over computer crime. + +Neidorf's alleged crimes centered around the E911 Document. +He was accused of having entered into a fraudulent scheme with the Prophet, +who, it will be recalled, was the Atlanta LoD member who had illicitly +copied the E911 Document from the BellSouth AIMSX system. + +The Prophet himself was also a co-defendant in the Neidorf case, +part-and-parcel of the alleged "fraud scheme" to "steal" BellSouth's +E911 Document (and to pass the Document across state lines, +which helped establish the Neidorf trial as a federal case). +The Prophet, in the spirit of full co-operation, had agreed +to testify against Neidorf. + +In fact, all three of the Atlanta crew stood ready to testify against Neidorf. +Their own federal prosecutors in Atlanta had charged the Atlanta Three with: +(a) conspiracy, (b) computer fraud, (c) wire fraud, (d) access device fraud, +and (e) interstate transportation of stolen property (Title 18, Sections 371, +1030, 1343, 1029, and 2314). + +Faced with this blizzard of trouble, Prophet and Leftist had ducked +any public trial and had pled guilty to reduced charges--one conspiracy +count apiece. Urvile had pled guilty to that odd bit of Section 1029 +which makes it illegal to possess "fifteen or more" illegal access devices +(in his case, computer passwords). And their sentences were scheduled +for September 14, 1990--well after the Neidorf trial. As witnesses, +they could presumably be relied upon to behave. + +Neidorf, however, was pleading innocent. Most everyone else caught up +in the crackdown had "cooperated fully" and pled guilty in hope +of reduced sentences. (Steve Jackson was a notable exception, +of course, and had strongly protested his innocence from the +very beginning. But Steve Jackson could not get a day in court-- +Steve Jackson had never been charged with any crime in the first place.) + +Neidorf had been urged to plead guilty. But Neidorf was a political science +major and was disinclined to go to jail for "fraud" when he had not made +any money, had not broken into any computer, and had been publishing +a magazine that he considered protected under the First Amendment. + +Neidorf's trial was the ONLY legal action of the entire Crackdown +that actually involved bringing the issues at hand out for a public test +in front of a jury of American citizens. + +Neidorf, too, had cooperated with investigators. He had voluntarily +handed over much of the evidence that had led to his own indictment. +He had already admitted in writing that he knew that the E911 Document +had been stolen before he had "published" it in Phrack--or, from the +prosecution's point of view, illegally transported stolen property by wire +in something purporting to be a "publication." + +But even if the "publication" of the E911 Document was not held to be a crime, +that wouldn't let Neidorf off the hook. Neidorf had still received +the E911 Document when Prophet had transferred it to him from Rich Andrews' +Jolnet node. On that occasion, it certainly hadn't been "published"-- +it was hacker booty, pure and simple, transported across state lines. + +The Chicago Task Force led a Chicago grand jury to indict Neidorf +on a set of charges that could have put him in jail for thirty years. +When some of these charges were successfully challenged before Neidorf +actually went to trial, the Chicago Task Force rearranged his +indictment so that he faced a possible jail term of over sixty years! +As a first offender, it was very unlikely that Neidorf would in fact +receive a sentence so drastic; but the Chicago Task Force clearly +intended to see Neidorf put in prison, and his conspiratorial "magazine" +put permanently out of commission. This was a federal case, and Neidorf +was charged with the fraudulent theft of property worth almost +eighty thousand dollars. + +William Cook was a strong believer in high-profile prosecutions +with symbolic overtones. He often published articles on his work +in the security trade press, arguing that "a clear message had +to be sent to the public at large and the computer community +in particular that unauthorized attacks on computers and the theft +of computerized information would not be tolerated by the courts." + +The issues were complex, the prosecution's tactics somewhat unorthodox, +but the Chicago Task Force had proved sure-footed to date. "Shadowhawk" +had been bagged on the wing in 1989 by the Task Force, and sentenced +to nine months in prison, and a $10,000 fine. The Shadowhawk case involved +charges under Section 1030, the "federal interest computer" section. + +Shadowhawk had not in fact been a devotee of "federal-interest" computers +per se. On the contrary, Shadowhawk, who owned an AT&T home computer, +seemed to cherish a special aggression toward AT&T. He had bragged on +the underground boards "Phreak Klass 2600" and "Dr. Ripco" of his skills +at raiding AT&T, and of his intention to crash AT&T's national phone system. +Shadowhawk's brags were noticed by Henry Kluepfel of Bellcore Security, +scourge of the outlaw boards, whose relations with the Chicago Task Force +were long and intimate. + +The Task Force successfully established that Section 1030 applied to +the teenage Shadowhawk, despite the objections of his defense attorney. +Shadowhawk had entered a computer "owned" by U.S. Missile Command +and merely "managed" by AT&T. He had also entered an AT&T computer +located at Robbins Air Force Base in Georgia. Attacking AT&T was +of "federal interest" whether Shadowhawk had intended it or not. + +The Task Force also convinced the court that a piece of AT&T +software that Shadowhawk had illicitly copied from Bell Labs, +the "Artificial Intelligence C5 Expert System," was worth a cool +one million dollars. Shadowhawk's attorney had argued that +Shadowhawk had not sold the program and had made no profit from +the illicit copying. And in point of fact, the C5 Expert System +was experimental software, and had no established market value +because it had never been on the market in the first place. +AT&T's own assessment of a "one million dollar" figure for its +own intangible property was accepted without challenge +by the court, however. And the court concurred with +the government prosecutors that Shadowhawk showed clear +"intent to defraud" whether he'd gotten any money or not. +Shadowhawk went to jail. + +The Task Force's other best-known triumph had been the conviction +and jailing of "Kyrie." Kyrie, a true denizen of the digital +criminal underground, was a 36-year-old Canadian woman, +convicted and jailed for telecommunications fraud in Canada. +After her release from prison, she had fled the wrath of Canada Bell +and the Royal Canadian Mounted Police, and eventually settled, +very unwisely, in Chicago. + +"Kyrie," who also called herself "Long Distance Information," +specialized in voice-mail abuse. She assembled large numbers +of hot long-distance codes, then read them aloud into a series +of corporate voice-mail systems. Kyrie and her friends were +electronic squatters in corporate voice-mail systems, +using them much as if they were pirate bulletin boards, +then moving on when their vocal chatter clogged the system +and the owners necessarily wised up. Kyrie's camp followers +were a loose tribe of some hundred and fifty phone-phreaks, +who followed her trail of piracy from machine to machine, +ardently begging for her services and expertise. + +Kyrie's disciples passed her stolen credit-card numbers, +in exchange for her stolen "long distance information." +Some of Kyrie's clients paid her off in cash, by scamming +credit-card cash advances from Western Union. + +Kyrie travelled incessantly, mostly through airline tickets +and hotel rooms that she scammed through stolen credit cards. +Tiring of this, she found refuge with a fellow female phone +phreak in Chicago. Kyrie's hostess, like a surprising number +of phone phreaks, was blind. She was also physically disabled. +Kyrie allegedly made the best of her new situation by applying for, +and receiving, state welfare funds under a false identity as +a qualified caretaker for the handicapped. + +Sadly, Kyrie's two children by a former marriage had also vanished +underground with her; these pre-teen digital refugees had no legal +American identity, and had never spent a day in school. + +Kyrie was addicted to technical mastery and enthralled by her own +cleverness and the ardent worship of her teenage followers. +This foolishly led her to phone up Gail Thackeray in Arizona, +to boast, brag, strut, and offer to play informant. +Thackeray, however, had already learned far more +than enough about Kyrie, whom she roundly despised +as an adult criminal corrupting minors, a "female Fagin." +Thackeray passed her tapes of Kyrie's boasts to the Secret Service. + +Kyrie was raided and arrested in Chicago in May 1989. +She confessed at great length and pled guilty. + +In August 1990, Cook and his Task Force colleague Colleen Coughlin +sent Kyrie to jail for 27 months, for computer and telecommunications fraud. +This was a markedly severe sentence by the usual wrist-slapping standards +of "hacker" busts. Seven of Kyrie's foremost teenage disciples were also +indicted and convicted. The Kyrie "high-tech street gang," as Cook +described it, had been crushed. Cook and his colleagues had been +the first ever to put someone in prison for voice-mail abuse. +Their pioneering efforts had won them attention and kudos. + +In his article on Kyrie, Cook drove the message home to the readers +of Security Management magazine, a trade journal for corporate +security professionals. The case, Cook said, and Kyrie's stiff sentence, +"reflect a new reality for hackers and computer crime victims in the +'90s. . . . Individuals and corporations who report computer +and telecommunications crimes can now expect that their cooperation +with federal law enforcement will result in meaningful punishment. +Companies and the public at large must report computer-enhanced +crimes if they want prosecutors and the course to protect their rights +to the tangible and intangible property developed and stored on computers." + +Cook had made it his business to construct this "new reality for hackers." +He'd also made it his business to police corporate property rights +to the intangible. + +Had the Electronic Frontier Foundation been a "hacker defense fund" +as that term was generally understood, they presumably would have stood up +for Kyrie. Her 1990 sentence did indeed send a "message" that federal heat +was coming down on "hackers." But Kyrie found no defenders at EFF, +or anywhere else, for that matter. EFF was not a bail-out fund +for electronic crooks. + +The Neidorf case paralleled the Shadowhawk case in certain ways. +The victim once again was allowed to set the value of the "stolen" property. +Once again Kluepfel was both investigator and technical advisor. +Once again no money had changed hands, but the "intent to defraud" was central. + +The prosecution's case showed signs of weakness early on. The Task Force +had originally hoped to prove Neidorf the center of a nationwide +Legion of Doom criminal conspiracy. The Phrack editors threw physical +get-togethers every summer, which attracted hackers from across the country; +generally two dozen or so of the magazine's favorite contributors and readers. +(Such conventions were common in the hacker community; 2600 Magazine, +for instance, held public meetings of hackers in New York, every month.) +LoD heavy-dudes were always a strong presence at these Phrack-sponsored +"Summercons." + +In July 1988, an Arizona hacker named "Dictator" attended Summercon +in Neidorf's home town of St. Louis. Dictator was one of Gail Thackeray's +underground informants; Dictator's underground board in Phoenix was +a sting operation for the Secret Service. Dictator brought an undercover +crew of Secret Service agents to Summercon. The agents bored spyholes +through the wall of Dictator's hotel room in St Louis, and videotaped +the frolicking hackers through a one-way mirror. As it happened, +however, nothing illegal had occurred on videotape, other than the +guzzling of beer by a couple of minors. Summercons were social events, +not sinister cabals. The tapes showed fifteen hours of raucous laughter, +pizza-gobbling, in-jokes and back-slapping. + +Neidorf's lawyer, Sheldon Zenner, saw the Secret Service tapes +before the trial. Zenner was shocked by the complete harmlessness +of this meeting, which Cook had earlier characterized as a sinister +interstate conspiracy to commit fraud. Zenner wanted to show the +Summercon tapes to the jury. It took protracted maneuverings +by the Task Force to keep the tapes from the jury as "irrelevant." + +The E911 Document was also proving a weak reed. It had originally +been valued at $79,449. Unlike Shadowhawk's arcane Artificial Intelligence +booty, the E911 Document was not software--it was written in English. +Computer-knowledgeable people found this value--for a twelve-page +bureaucratic document--frankly incredible. In his "Crime and Puzzlement" +manifesto for EFF, Barlow commented: "We will probably never know how +this figure was reached or by whom, though I like to imagine an appraisal +team consisting of Franz Kafka, Joseph Heller, and Thomas Pynchon." + +As it happened, Barlow was unduly pessimistic. The EFF did, in fact, +eventually discover exactly how this figure was reached, and by whom-- +but only in 1991, long after the Neidorf trial was over. + +Kim Megahee, a Southern Bell security manager, +had arrived at the document's value by simply adding up +the "costs associated with the production" of the E911 Document. +Those "costs" were as follows: + +1. A technical writer had been hired to research and write the E911 Document. + 200 hours of work, at $35 an hour, cost : $7,000. A Project Manager had + overseen the technical writer. 200 hours, at $31 an hour, made: $6,200. + +2. A week of typing had cost $721 dollars. A week of formatting had + cost $721. A week of graphics formatting had cost $742. + +3. Two days of editing cost $367. + +4. A box of order labels cost five dollars. + +5. Preparing a purchase order for the Document, including typing + and the obtaining of an authorizing signature from within the + BellSouth bureaucracy, cost $129. + +6. Printing cost $313. Mailing the Document to fifty people + took fifty hours by a clerk, and cost $858. + +7. Placing the Document in an index took two clerks an hour each, + totalling $43. + +Bureaucratic overhead alone, therefore, was alleged to have cost +a whopping $17,099. According to Mr. Megahee, the typing +of a twelve-page document had taken a full week. Writing it +had taken five weeks, including an overseer who apparently +did nothing else but watch the author for five weeks. +Editing twelve pages had taken two days. Printing and mailing +an electronic document (which was already available on the +Southern Bell Data Network to any telco employee who needed it), +had cost over a thousand dollars. + +But this was just the beginning. There were also the HARDWARE EXPENSES. +Eight hundred fifty dollars for a VT220 computer monitor. +THIRTY-ONE THOUSAND DOLLARS for a sophisticated VAXstation II computer. +Six thousand dollars for a computer printer. TWENTY-TWO THOUSAND DOLLARS +for a copy of "Interleaf" software. Two thousand five hundred dollars +for VMS software. All this to create the twelve-page Document. + +Plus ten percent of the cost of the software and the hardware, for maintenance. +(Actually, the ten percent maintenance costs, though mentioned, had been left +off the final $79,449 total, apparently through a merciful oversight). + +Mr. Megahee's letter had been mailed directly to William Cook himself, +at the office of the Chicago federal attorneys. The United States Government +accepted these telco figures without question. + +As incredulity mounted, the value of the E911 Document was officially +revised downward. This time, Robert Kibler of BellSouth Security +estimated the value of the twelve pages as a mere $24,639.05--based, +purportedly, on "R&D costs." But this specific estimate, +right down to the nickel, did not move the skeptics at all; +in fact it provoked open scorn and a torrent of sarcasm. + +The financial issues concerning theft of proprietary information +have always been peculiar. It could be argued that BellSouth +had not "lost" its E911 Document at all in the first place, +and therefore had not suffered any monetary damage from this "theft." +And Sheldon Zenner did in fact argue this at Neidorf's trial-- +that Prophet's raid had not been "theft," but was better understood +as illicit copying. + +The money, however, was not central to anyone's true purposes in this trial. +It was not Cook's strategy to convince the jury that the E911 Document +was a major act of theft and should be punished for that reason alone. +His strategy was to argue that the E911 Document was DANGEROUS. +It was his intention to establish that the E911 Document was "a road-map" +to the Enhanced 911 System. Neidorf had deliberately and recklessly +distributed a dangerous weapon. Neidorf and the Prophet did not care +(or perhaps even gloated at the sinister idea) that the E911 Document +could be used by hackers to disrupt 911 service, "a life line for every +person certainly in the Southern Bell region of the United States, +and indeed, in many communities throughout the United States," +in Cook's own words. Neidorf had put people's lives in danger. + +In pre-trial maneuverings, Cook had established that the E911 Document +was too hot to appear in the public proceedings of the Neidorf trial. +The JURY ITSELF would not be allowed to ever see this Document, +lest it slip into the official court records, and thus into the hands +of the general public, and, thus, somehow, to malicious hackers +who might lethally abuse it. + +Hiding the E911 Document from the jury may have been a +clever legal maneuver, but it had a severe flaw. There were, +in point of fact, hundreds, perhaps thousands, of people, +already in possession of the E911 Document, just as Phrack +had published it. Its true nature was already obvious +to a wide section of the interested public (all of whom, +by the way, were, at least theoretically, party to +a gigantic wire-fraud conspiracy). Most everyone +in the electronic community who had a modem and any +interest in the Neidorf case already had a copy of the Document. +It had already been available in Phrack for over a year. + +People, even quite normal people without any particular +prurient interest in forbidden knowledge, did not shut their eyes +in terror at the thought of beholding a "dangerous" document +from a telephone company. On the contrary, they tended to trust +their own judgement and simply read the Document for themselves. +And they were not impressed. + +One such person was John Nagle. Nagle was a forty-one-year-old +professional programmer with a masters' degree in computer science +from Stanford. He had worked for Ford Aerospace, where he had invented +a computer-networking technique known as the "Nagle Algorithm," +and for the prominent Californian computer-graphics firm "Autodesk," +where he was a major stockholder. + +Nagle was also a prominent figure on the Well, much respected +for his technical knowledgeability. + +Nagle had followed the civil-liberties debate closely, +for he was an ardent telecommunicator. He was no particular friend +of computer intruders, but he believed electronic publishing +had a great deal to offer society at large, and attempts +to restrain its growth, or to censor free electronic expression, +strongly roused his ire. + +The Neidorf case, and the E911 Document, were both being discussed +in detail on the Internet, in an electronic publication called Telecom Digest. +Nagle, a longtime Internet maven, was a regular reader of Telecom Digest. +Nagle had never seen a copy of Phrack, but the implications of the case +disturbed him. + +While in a Stanford bookstore hunting books on robotics, +Nagle happened across a book called The Intelligent Network. +Thumbing through it at random, Nagle came across an entire chapter +meticulously detailing the workings of E911 police emergency systems. +This extensive text was being sold openly, and yet in Illinois +a young man was in danger of going to prison for publishing +a thin six-page document about 911 service. + +Nagle made an ironic comment to this effect in Telecom Digest. +From there, Nagle was put in touch with Mitch Kapor, +and then with Neidorf's lawyers. + +Sheldon Zenner was delighted to find a computer telecommunications expert +willing to speak up for Neidorf, one who was not a wacky teenage "hacker." +Nagle was fluent, mature, and respectable; he'd once had a federal +security clearance. + +Nagle was asked to fly to Illinois to join the defense team. + +Having joined the defense as an expert witness, Nagle read the entire +E911 Document for himself. He made his own judgement about its potential +for menace. + +The time has now come for you yourself, the reader, to have a look +at the E911 Document. This six-page piece of work was the pretext +for a federal prosecution that could have sent an electronic publisher +to prison for thirty, or even sixty, years. It was the pretext +for the search and seizure of Steve Jackson Games, a legitimate publisher +of printed books. It was also the formal pretext for the search +and seizure of the Mentor's bulletin board, "Phoenix Project," +and for the raid on the home of Erik Bloodaxe. It also had much +to do with the seizure of Richard Andrews' Jolnet node +and the shutdown of Charles Boykin's AT&T node. +The E911 Document was the single most important piece +of evidence in the Hacker Crackdown. There can be no real +and legitimate substitute for the Document itself. + + +==Phrack Inc.== + +Volume Two, Issue 24, File 5 of 13 + +Control Office Administration +Of Enhanced 911 Services For +Special Services and Account Centers + +by the Eavesdropper + +March, 1988 + + +Description of Service +~~~~~~~~~~~~~~~~~~~~~ +The control office for Emergency 911 service is assigned in +accordance with the existing standard guidelines to one of +the following centers: + +o Special Services Center (SSC) +o Major Accounts Center (MAC) +o Serving Test Center (STC) +o Toll Control Center (TCC) + +The SSC/MAC designation is used in this document interchangeably +for any of these four centers. The Special Services Centers (SSCs) +or Major Account Centers (MACs) have been designated as the trouble +reporting contact for all E911 customer (PSAP) reported troubles. +Subscribers who have trouble on an E911 call will continue +to contact local repair service (CRSAB) who will refer the +trouble to the SSC/MAC, when appropriate. + +Due to the critical nature of E911 service, the control +and timely repair of troubles is demanded. As the primary +E911 customer contact, the SSC/MAC is in the unique position +to monitor the status of the trouble and insure its resolution. + +System Overview +~~~~~~~~~~~~~~ +The number 911 is intended as a nationwide universal +telephone number which provides the public with direct +access to a Public Safety Answering Point (PSAP). A PSAP +is also referred to as an Emergency Service Bureau (ESB). +A PSAP is an agency or facility which is authorized by a +municipality to receive and respond to police, fire and/or +ambulance services. One or more attendants are located +at the PSAP facilities to receive and handle calls of an +emergency nature in accordance with the local municipal +requirements. + +An important advantage of E911 emergency service is +improved (reduced) response times for emergency +services. Also close coordination among agencies +providing various emergency services is a valuable +capability provided by E911 service. + +1A ESS is used as the tandem office for the E911 network to +route all 911 calls to the correct (primary) PSAP designated +to serve the calling station. The E911 feature was +developed primarily to provide routing to the correct PSAP +for all 911 calls. Selective routing allows a 911 call +originated from a particular station located in a particular +district, zone, or town, to be routed to the primary PSAP +designated to serve that customer station regardless of +wire center boundaries. Thus, selective routing eliminates +the problem of wire center boundaries not coinciding with +district or other political boundaries. + +The services available with the E911 feature include: + +Forced Disconnect Default Routing +Alternative Routing Night Service +Selective Routing Automatic Number +Identification (ANI) +Selective Transfer Automatic Location +Identification (ALI) + + +Preservice/Installation Guidelines +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +When a contract for an E911 system has been signed, it is +the responsibility of Network Marketing to establish an +implementation/cutover committee which should include +a representative from the SSC/MAC. Duties of the E911 +Implementation Team include coordination of all phases +of the E911 system deployment and the formation of an +on-going E911 maintenance subcommittee. + +Marketing is responsible for providing the following +customer specific information to the SSC/MAC prior to +the start of call through testing: + +o All PSAP's (name, address, local contact) +o All PSAP circuit ID's +o 1004 911 service request including PSAP details on each PSAP + (1004 Section K, L, M) +o Network configuration +o Any vendor information (name, telephone number, equipment) + +The SSC/MAC needs to know if the equipment and sets +at the PSAP are maintained by the BOCs, an independent +company, or an outside vendor, or any combination. +This information is then entered on the PSAP profile sheets +and reviewed quarterly for changes, additions and deletions. + +Marketing will secure the Major Account Number (MAN) +and provide this number to Corporate Communications +so that the initial issue of the service orders carry +the MAN and can be tracked by the SSC/MAC via CORDNET. +PSAP circuits are official services by definition. + +All service orders required for the installation of the E911 +system should include the MAN assigned to the city/county +which has purchased the system. + +In accordance with the basic SSC/MAC strategy for provisioning, +the SSC/MAC will be Overall Control Office (OCO) for all Node +to PSAP circuits (official services) and any other services +for this customer. Training must be scheduled for all SSC/MAC +involved personnel during the pre-service stage of the project. + +The E911 Implementation Team will form the on-going +maintenance subcommittee prior to the initial +implementation of the E911 system. This sub-committee +will establish post implementation quality assurance +procedures to ensure that the E911 system continues to +provide quality service to the customer. +Customer/Company training, trouble reporting interfaces +for the customer, telephone company and any involved +independent telephone companies needs to be addressed +and implemented prior to E911 cutover. These functions +can be best addressed by the formation of a sub- +committee of the E911 Implementation Team to set up +guidelines for and to secure service commitments of +interfacing organizations. A SSC/MAC supervisor should +chair this subcommittee and include the following +organizations: + +1) Switching Control Center + - E911 translations + - Trunking + - End office and Tandem office hardware/software +2) Recent Change Memory Administration Center + - Daily RC update activity for TN/ESN translations + - Processes validity errors and rejects +3) Line and Number Administration + - Verification of TN/ESN translations +4) Special Service Center/Major Account Center + - Single point of contact for all PSAP and Node to host troubles + - Logs, tracks & statusing of all trouble reports + - Trouble referral, follow up, and escalation + - Customer notification of status and restoration + - Analyzation of "chronic" troubles + - Testing, installation and maintenance of E911 circuits +5) Installation and Maintenance (SSIM/I&M) + - Repair and maintenance of PSAP equipment and Telco owned sets +6) Minicomputer Maintenance Operations Center + - E911 circuit maintenance (where applicable) +7) Area Maintenance Engineer + - Technical assistance on voice (CO-PSAP) network related E911 troubles + + +Maintenance Guidelines +~~~~~~~~~~~~~~~~~~~~~ +The CCNC will test the Node circuit from the 202T at the +Host site to the 202T at the Node site. Since Host to Node +(CCNC to MMOC) circuits are official company services, +the CCNC will refer all Node circuit troubles to the +SSC/MAC. The SSC/MAC is responsible for the testing +and follow up to restoration of these circuit troubles. + +Although Node to PSAP circuit are official services, the +MMOC will refer PSAP circuit troubles to the appropriate +SSC/MAC. The SSC/MAC is responsible for testing and +follow up to restoration of PSAP circuit troubles. + +The SSC/MAC will also receive reports from +CRSAB/IMC(s) on subscriber 911 troubles when they are +not line troubles. The SSC/MAC is responsible for testing +and restoration of these troubles. + +Maintenance responsibilities are as follows: + +SCC@ Voice Network (ANI to PSAP) +@SCC responsible for tandem switch + +SSIM/I&M PSAP Equipment (Modems, CIU's, sets) +Vendor PSAP Equipment (when CPE) +SSC/MAC PSAP to Node circuits, and tandem to + PSAP voice circuits (EMNT) +MMOC Node site (Modems, cables, etc) + +Note: All above work groups are required to resolve troubles +by interfacing with appropriate work groups for resolution. + +The Switching Control Center (SCC) is responsible for +E911/1AESS translations in tandem central offices. +These translations route E911 calls, selective transfer, +default routing, speed calling, etc., for each PSAP. +The SCC is also responsible for troubleshooting on +the voice network (call originating to end office tandem equipment). + +For example, ANI failures in the originating offices would +be a responsibility of the SCC. + +Recent Change Memory Administration Center (RCMAC) performs +the daily tandem translation updates (recent change) +for routing of individual telephone numbers. + +Recent changes are generated from service order activity +(new service, address changes, etc.) and compiled into +a daily file by the E911 Center (ALI/DMS E911 Computer). + +SSIM/I&M is responsible for the installation and repair of +PSAP equipment. PSAP equipment includes ANI Controller, +ALI Controller, data sets, cables, sets, and other peripheral +equipment that is not vendor owned. SSIM/I&M is responsible +for establishing maintenance test kits, complete with spare parts +for PSAP maintenance. This includes test gear, data sets, +and ANI/ALI Controller parts. + +Special Services Center (SSC) or Major Account Center +(MAC) serves as the trouble reporting contact for all +(PSAP) troubles reported by customer. The SSC/MAC +refers troubles to proper organizations for handling and +tracks status of troubles, escalating when necessary. +The SSC/MAC will close out troubles with customer. +The SSC/MAC will analyze all troubles and tracks "chronic" +PSAP troubles. + +Corporate Communications Network Center (CCNC) will +test and refer troubles on all node to host circuits. +All E911 circuits are classified as official company property. + +The Minicomputer Maintenance Operations Center +(MMOC) maintains the E911 (ALI/DMS) computer +hardware at the Host site. This MMOC is also responsible +for monitoring the system and reporting certain PSAP +and system problems to the local MMOC's, SCC's or +SSC/MAC's. The MMOC personnel also operate software +programs that maintain the TN data base under the +direction of the E911 Center. The maintenance of the +NODE computer (the interface between the PSAP and the +ALI/DMS computer) is a function of the MMOC at the +NODE site. The MMOC's at the NODE sites may also be +involved in the testing of NODE to Host circuits. +The MMOC will also assist on Host to PSAP and data network +related troubles not resolved through standard trouble +clearing procedures. + +Installation And Maintenance Center (IMC) is responsible +for referral of E911 subscriber troubles that are not subscriber +line problems. + +E911 Center - Performs the role of System Administration +and is responsible for overall operation of the E911 +computer software. The E911 Center does A-Z trouble +analysis and provides statistical information on the +performance of the system. + +This analysis includes processing PSAP inquiries (trouble +reports) and referral of network troubles. The E911 Center +also performs daily processing of tandem recent change +and provides information to the RCMAC for tandem input. +The E911 Center is responsible for daily processing +of the ALI/DMS computer data base and provides error files, +etc. to the Customer Services department for investigation and correction. +The E911 Center participates in all system implementations and on-going +maintenance effort and assists in the development of procedures, +training and education of information to all groups. + +Any group receiving a 911 trouble from the SSC/MAC should +close out the trouble with the SSC/MAC or provide a status +if the trouble has been referred to another group. +This will allow the SSC/MAC to provide a status back +to the customer or escalate as appropriate. + +Any group receiving a trouble from the Host site (MMOC +or CCNC) should close the trouble back to that group. + +The MMOC should notify the appropriate SSC/MAC +when the Host, Node, or all Node circuits are down so that +the SSC/MAC can reply to customer reports that may be +called in by the PSAPs. This will eliminate duplicate +reporting of troubles. On complete outages the MMOC +will follow escalation procedures for a Node after two (2) +hours and for a PSAP after four (4) hours. Additionally the +MMOC will notify the appropriate SSC/MAC when the +Host, Node, or all Node circuits are down. + +The PSAP will call the SSC/MAC to report E911 troubles. +The person reporting the E911 trouble may not have a +circuit I.D. and will therefore report the PSAP name and +address. Many PSAP troubles are not circuit specific. In +those instances where the caller cannot provide a circuit +I.D., the SSC/MAC will be required to determine the +circuit I.D. using the PSAP profile. Under no circumstances +will the SSC/MAC Center refuse to take the trouble. +The E911 trouble should be handled as quickly as possible, +with the SSC/MAC providing as much assistance as +possible while taking the trouble report from the caller. + +The SSC/MAC will screen/test the trouble to determine the +appropriate handoff organization based on the following criteria: + +PSAP equipment problem: SSIM/I&M +Circuit problem: SSC/MAC +Voice network problem: SCC (report trunk group number) +Problem affecting multiple PSAPs (No ALI report from +all PSAPs): Contact the MMOC to check for NODE or +Host computer problems before further testing. + +The SSC/MAC will track the status of reported troubles +and escalate as appropriate. The SSC/MAC will close out +customer/company reports with the initiating contact. +Groups with specific maintenance responsibilities, +defined above, will investigate "chronic" troubles upon +request from the SSC/MAC and the ongoing maintenance subcommittee. + +All "out of service" E911 troubles are priority one type reports. +One link down to a PSAP is considered a priority one trouble +and should be handled as if the PSAP was isolated. + +The PSAP will report troubles with the ANI controller, ALI +controller or set equipment to the SSC/MAC. + +NO ANI: Where the PSAP reports NO ANI (digital +display screen is blank) ask if this condition exists on all +screens and on all calls. It is important to differentiate +between blank screens and screens displaying 911-00XX, +or all zeroes. + +When the PSAP reports all screens on all calls, ask if there +is any voice contact with callers. If there is no voice +contact the trouble should be referred to the SCC +immediately since 911 calls are not getting through which +may require alternate routing of calls to another PSAP. + +When the PSAP reports this condition on all screens +but not all calls and has voice contact with callers, +the report should be referred to SSIM/I&M for dispatch. +The SSC/MAC should verify with the SCC that ANI +is pulsing before dispatching SSIM. + +When the PSAP reports this condition on one screen for +all calls (others work fine) the trouble should be referred +to SSIM/I&M for dispatch, because the trouble is isolated to +one piece of equipment at the customer premise. + +An ANI failure (i.e. all zeroes) indicates that the ANI has +not been received by the PSAP from the tandem office or +was lost by the PSAP ANI controller. The PSAP may +receive "02" alarms which can be caused by the ANI +controller logging more than three all zero failures on the +same trunk. The PSAP has been instructed to report this +condition to the SSC/MAC since it could indicate an +equipment trouble at the PSAP which might be affecting +all subscribers calling into the PSAP. When all zeroes are +being received on all calls or "02" alarms continue, a tester +should analyze the condition to determine the appropriate +action to be taken. The tester must perform cooperative +testing with the SCC when there appears to be a problem +on the Tandem-PSAP trunks before requesting dispatch. + +When an occasional all zero condition is reported, +the SSC/MAC should dispatch SSIM/I&M to routine +equipment on a "chronic" troublesweep. + +The PSAPs are instructed to report incidental ANI failures +to the BOC on a PSAP inquiry trouble ticket (paper) that +is sent to the Customer Services E911 group and forwarded +to E911 center when required. This usually involves only a +particular telephone number and is not a condition that +would require a report to the SSC/MAC. Multiple ANI +failures which our from the same end office (XX denotes +end office), indicate a hard trouble condition may exist +in the end office or end office tandem trunks. The PSAP will +report this type of condition to the SSC/MAC and the +SSC/MAC should refer the report to the SCC responsible +for the tandem office. NOTE: XX is the ESCO (Emergency +Service Number) associated with the incoming 911 trunks +into the tandem. It is important that the C/MAC tell the +SCC what is displayed at the PSAP (i.e. 911-0011) which +indicates to the SCC which end office is in trouble. + +Note: It is essential that the PSAP fill out inquiry form +on every ANI failure. + +The PSAP will report a trouble any time an address is not +received on an address display (screen blank) E911 call. +(If a record is not in the 911 data base or an ANI failure +is encountered, the screen will provide a display noticing +such condition). The SSC/MAC should verify with the PSAP +whether the NO ALI condition is on one screen or all screens. + +When the condition is on one screen (other screens +receive ALI information) the SSC/MAC will request +SSIM/I&M to dispatch. + +If no screens are receiving ALI information, there is usually +a circuit trouble between the PSAP and the Host computer. +The SSC/MAC should test the trouble and refer for restoral. + +Note: If the SSC/MAC receives calls from multiple +PSAP's, all of which are receiving NO ALI, there is a +problem with the Node or Node to Host circuits or the +Host computer itself. Before referring the trouble the +SSC/MAC should call the MMOC to inquire if the Node +or Host is in trouble. + +Alarm conditions on the ANI controller digital display at +the PSAP are to be reported by the PSAP's. These alarms +can indicate various trouble conditions so the SSC/MAC +should ask the PSAP if any portion of the E911 system +is not functioning properly. + +The SSC/MAC should verify with the PSAP attendant that +the equipment's primary function is answering E911 calls. +If it is, the SSC/MAC should request a dispatch SSIM/I&M. +If the equipment is not primarily used for E911, +then the SSC/MAC should advise PSAP to contact their CPE vendor. + +Note: These troubles can be quite confusing when the +PSAP has vendor equipment mixed in with equipment +that the BOC maintains. The Marketing representative +should provide the SSC/MAC information concerning any +unusual or exception items where the PSAP should +contact their vendor. This information should be included +in the PSAP profile sheets. + +ANI or ALI controller down: When the host computer sees +the PSAP equipment down and it does not come back up, +the MMOC will report the trouble to the SSC/MAC; +the equipment is down at the PSAP, a dispatch will be required. + +PSAP link (circuit) down: The MMOC will provide the +SSC/MAC with the circuit ID that the Host computer +indicates in trouble. Although each PSAP has two circuits, +when either circuit is down the condition must be treated +as an emergency since failure of the second circuit will +cause the PSAP to be isolated. + +Any problems that the MMOC identifies from the Node +location to the Host computer will be handled directly +with the appropriate MMOC(s)/CCNC. + +Note: The customer will call only when a problem is +apparent to the PSAP. When only one circuit is down to +the PSAP, the customer may not be aware there is a +trouble, even though there is one link down, +notification should appear on the PSAP screen. +Troubles called into the SSC/MAC from the MMOC +or other company employee should not be closed out +by calling the PSAP since it may result in the +customer responding that they do not have a trouble. +These reports can only be closed out by receiving +information that the trouble was fixed and by checking +with the company employee that reported the trouble. +The MMOC personnel will be able to verify that the +trouble has cleared by reviewing a printout from the host. + +When the CRSAB receives a subscriber complaint +(i.e., cannot dial 911) the RSA should obtain as much +information as possible while the customer is on the line. + +For example, what happened when the subscriber dialed 911? +The report is automatically directed to the IMC for subscriber line testing. +When no line trouble is found, the IMC will refer the trouble condition +to the SSC/MAC. The SSC/MAC will contact Customer Services E911 Group +and verify that the subscriber should be able to call 911 and obtain the ESN. +The SSC/MAC will verify the ESN via 2SCCS. When both verifications match, +the SSC/MAC will refer the report to the SCC responsible for the 911 tandem +office for investigation and resolution. The MAC is responsible for tracking +the trouble and informing the IMC when it is resolved. + + +For more information, please refer to E911 Glossary of Terms. +End of Phrack File +_____________________________________ + + +The reader is forgiven if he or she was entirely unable to read +this document. John Perry Barlow had a great deal of fun at its expense, +in "Crime and Puzzlement:" "Bureaucrat-ese of surpassing opacity. . . . +To read the whole thing straight through without entering coma requires +either a machine or a human who has too much practice thinking like one. +Anyone who can understand it fully and fluidly had altered his consciousness +beyond the ability to ever again read Blake, Whitman, or Tolstoy. . . . +the document contains little of interest to anyone who is not a student +of advanced organizational sclerosis." + +With the Document itself to hand, however, exactly as it was published +(in its six-page edited form) in Phrack, the reader may be able to verify +a few statements of fact about its nature. First, there is no software, +no computer code, in the Document. It is not computer-programming language +like FORTRAN or C++, it is English; all the sentences have nouns and verbs +and punctuation. It does not explain how to break into the E911 system. +It does not suggest ways to destroy or damage the E911 system. + +There are no access codes in the Document. There are no computer passwords. +It does not explain how to steal long distance service. It does not explain +how to break in to telco switching stations. There is nothing in it about +using a personal computer or a modem for any purpose at all, good or bad. + +Close study will reveal that this document is not about machinery. +The E911 Document is about ADMINISTRATION. It describes how one creates +and administers certain units of telco bureaucracy: +Special Service Centers and Major Account Centers (SSC/MAC). +It describes how these centers should distribute responsibility +for the E911 service, to other units of telco bureaucracy, +in a chain of command, a formal hierarchy. It describes +who answers customer complaints, who screens calls, +who reports equipment failures, who answers those reports, +who handles maintenance, who chairs subcommittees, +who gives orders, who follows orders, WHO tells WHOM what to do. +The Document is not a "roadmap" to computers. +The Document is a roadmap to PEOPLE. + +As an aid to breaking into computer systems, the Document is USELESS. +As an aid to harassing and deceiving telco people, however, the Document +might prove handy (especially with its Glossary, which I have not included). +An intense and protracted study of this Document and its Glossary, +combined with many other such documents, might teach one to speak like +a telco employee. And telco people live by SPEECH--they live by phone +communication. If you can mimic their language over the phone, +you can "social-engineer" them. If you can con telco people, you can +wreak havoc among them. You can force them to no longer trust one another; +you can break the telephonic ties that bind their community; you can make +them paranoid. And people will fight harder to defend their community +than they will fight to defend their individual selves. + +This was the genuine, gut-level threat posed by Phrack magazine. +The real struggle was over the control of telco language, +the control of telco knowledge. It was a struggle to defend the social +"membrane of differentiation" that forms the walls of the telco +community's ivory tower --the special jargon that allows telco +professionals to recognize one another, and to exclude charlatans, +thieves, and upstarts. And the prosecution brought out this fact. +They repeatedly made reference to the threat posed to telco professionals +by hackers using "social engineering." + +However, Craig Neidorf was not on trial for learning to speak like +a professional telecommunications expert. Craig Neidorf was on trial +for access device fraud and transportation of stolen property. +He was on trial for stealing a document that was purportedly +highly sensitive and purportedly worth tens of thousands of dollars. + +# + +John Nagle read the E911 Document. He drew his own conclusions. +And he presented Zenner and his defense team with an overflowing box +of similar material, drawn mostly from Stanford University's +engineering libraries. During the trial, the defense team--Zenner, +half-a-dozen other attorneys, Nagle, Neidorf, and computer-security +expert Dorothy Denning, all pored over the E911 Document line-by-line. + +On the afternoon of July 25, 1990, Zenner began to cross-examine +a woman named Billie Williams, a service manager for Southern Bell +in Atlanta. Ms. Williams had been responsible for the E911 Document. +(She was not its author--its original "author" was a Southern Bell +staff manager named Richard Helms. However, Mr. Helms should not bear +the entire blame; many telco staff people and maintenance personnel +had amended the Document. It had not been so much "written" by a +single author, as built by committee out of concrete-blocks of jargon.) + +Ms. Williams had been called as a witness for the prosecution, +and had gamely tried to explain the basic technical structure +of the E911 system, aided by charts. + +Now it was Zenner's turn. He first established that the +"proprietary stamp" that BellSouth had used on the E911 Document +was stamped on EVERY SINGLE DOCUMENT that BellSouth wrote-- +THOUSANDS of documents. "We do not publish anything other +than for our own company," Ms. Williams explained. +"Any company document of this nature is considered proprietary." +Nobody was in charge of singling out special high-security publications +for special high-security protection. They were ALL special, +no matter how trivial, no matter what their subject matter-- +the stamp was put on as soon as any document was written, +and the stamp was never removed. + +Zenner now asked whether the charts she had been using to explain +the mechanics of E911 system were "proprietary," too. +Were they PUBLIC INFORMATION, these charts, all about PSAPs, +ALIs, nodes, local end switches? Could he take the charts out +in the street and show them to anybody, "without violating +some proprietary notion that BellSouth has?" + +Ms Williams showed some confusion, but finally areed that the charts were, +in fact, public. + +"But isn't this what you said was basically what appeared in Phrack?" + +Ms. Williams denied this. + +Zenner now pointed out that the E911 Document as published in Phrack +was only half the size of the original E911 Document (as Prophet +had purloined it). Half of it had been deleted--edited by Neidorf. + +Ms. Williams countered that "Most of the information that is +in the text file is redundant." + +Zenner continued to probe. Exactly what bits of knowledge in the Document +were, in fact, unknown to the public? Locations of E911 computers? +Phone numbers for telco personnel? Ongoing maintenance subcommittees? +Hadn't Neidorf removed much of this? + +Then he pounced. "Are you familiar with Bellcore Technical Reference +Document TR-TSY-000350?" It was, Zenner explained, officially titled +"E911 Public Safety Answering Point Interface Between 1-1AESS Switch +and Customer Premises Equipment." It contained highly detailed +and specific technical information about the E911 System. +It was published by Bellcore and publicly available for about $20. + +He showed the witness a Bellcore catalog which listed thousands +of documents from Bellcore and from all the Baby Bells, BellSouth included. +The catalog, Zenner pointed out, was free. Anyone with a credit card +could call the Bellcore toll-free 800 number and simply order any +of these documents, which would be shipped to any customer without question. +Including, for instance, "BellSouth E911 Service Interfaces to +Customer Premises Equipment at a Public Safety Answering Point." + +Zenner gave the witness a copy of "BellSouth E911 Service Interfaces," +which cost, as he pointed out, $13, straight from the catalog. +"Look at it carefully," he urged Ms. Williams, "and tell me +if it doesn't contain about twice as much detailed information +about the E911 system of BellSouth than appeared anywhere in Phrack." + +"You want me to. . . ." Ms. Williams trailed off. "I don't understand." + +"Take a careful look," Zenner persisted. "Take a look at that document, +and tell me when you're done looking at it if, indeed, it doesn't contain +much more detailed information about the E911 system than appeared in Phrack." + +"Phrack wasn't taken from this," Ms. Williams said. + +"Excuse me?" said Zenner. + +"Phrack wasn't taken from this." + +"I can't hear you," Zenner said. + +"Phrack was not taken from this document. I don't understand +your question to me." + +"I guess you don't," Zenner said. + +At this point, the prosecution's case had been gutshot. +Ms. Williams was distressed. Her confusion was quite genuine. +Phrack had not been taken from any publicly available Bellcore document. +Phrack's E911 Document had been stolen from her own company's computers, +from her own company's text files, that her own colleagues had written, +and revised, with much labor. + +But the "value" of the Document had been blown to smithereens. +It wasn't worth eighty grand. According to Bellcore it was worth +thirteen bucks. And the looming menace that it supposedly posed +had been reduced in instants to a scarecrow. Bellcore itself +was selling material far more detailed and "dangerous," +to anybody with a credit card and a phone. + +Actually, Bellcore was not giving this information to just anybody. +They gave it to ANYBODY WHO ASKED, but not many did ask. +Not many people knew that Bellcore had a free catalog and an 800 number. +John Nagle knew, but certainly the average teenage phreak didn't know. +"Tuc," a friend of Neidorf's and sometime Phrack contributor, knew, +and Tuc had been very helpful to the defense, behind the scenes. +But the Legion of Doom didn't know--otherwise, they would never +have wasted so much time raiding dumpsters. Cook didn't know. +Foley didn't know. Kluepfel didn't know. The right hand +of Bellcore knew not what the left hand was doing. The right +hand was battering hackers without mercy, while the left hand +was distributing Bellcore's intellectual property to anybody +who was interested in telephone technical trivia--apparently, +a pathetic few. + +The digital underground was so amateurish and poorly organized +that they had never discovered this heap of unguarded riches. +The ivory tower of the telcos was so wrapped-up in the fog +of its own technical obscurity that it had left all the +windows open and flung open the doors. No one had even noticed. + +Zenner sank another nail in the coffin. He produced a printed issue +of Telephone Engineer & Management, a prominent industry journal +that comes out twice a month and costs $27 a year. This particular issue +of TE&M, called "Update on 911," featured a galaxy of technical details +on 911 service and a glossary far more extensive than Phrack's. + +The trial rumbled on, somehow, through its own momentum. +Tim Foley testified about his interrogations of Neidorf. +Neidorf's written admission that he had known the E911 Document +was pilfered was officially read into the court record. + +An interesting side issue came up: "Terminus" had once passed Neidorf +a piece of UNIX AT&T software, a log-in sequence, that had been cunningly +altered so that it could trap passwords. The UNIX software itself was +illegally copied AT&T property, and the alterations "Terminus" had made to it, +had transformed it into a device for facilitating computer break-ins. Terminus +himself would eventually plead guilty to theft of this piece of software, +and the Chicago group would send Terminus to prison for it. But it was +of dubious relevance in the Neidorf case. Neidorf hadn't written the program. +He wasn't accused of ever having used it. And Neidorf wasn't being charged +with software theft or owning a password trapper. + +On the next day, Zenner took the offensive. The civil libertarians +now had their own arcane, untried legal weaponry to launch into action-- +the Electronic Communications Privacy Act of 1986, 18 US Code, +Section 2701 et seq. Section 2701 makes it a crime to intentionally +access without authorization a facility in which an electronic communication +service is provided--it is, at heart, an anti-bugging and anti-tapping law, +intended to carry the traditional protections of telephones into other +electronic channels of communication. While providing penalties for amateur +snoops, however, Section 2703 of the ECPA also lays some formal difficulties +on the bugging and tapping activities of police. + +The Secret Service, in the person of Tim Foley, had served Richard Andrews +with a federal grand jury subpoena, in their pursuit of Prophet, +the E911 Document, and the Terminus software ring. But according to +the Electronic Communications Privacy Act, a "provider of remote +computing service" was legally entitled to "prior notice" from +the government if a subpoena was used. Richard Andrews and his +basement UNIX node, Jolnet, had not received any "prior notice." +Tim Foley had purportedly violated the ECPA and committed +an electronic crime! Zenner now sought the judge's permission +to cross-examine Foley on the topic of Foley's own electronic misdeeds. + +Cook argued that Richard Andrews' Jolnet was a privately owned +bulletin board, and not within the purview of ECPA. Judge Bua +granted the motion of the government to prevent cross-examination +on that point, and Zenner's offensive fizzled. This, however, +was the first direct assault on the legality of the actions +of the Computer Fraud and Abuse Task Force itself-- +the first suggestion that they themselves had broken the law, +and might, perhaps, be called to account. + +Zenner, in any case, did not really need the ECPA. +Instead, he grilled Foley on the glaring contradictions in +the supposed value of the E911 Document. He also brought up +the embarrassing fact that the supposedly red-hot E911 Document +had been sitting around for months, in Jolnet, with Kluepfel's knowledge, +while Kluepfel had done nothing about it. + +In the afternoon, the Prophet was brought in to testify +for the prosecution. (The Prophet, it will be recalled, +had also been indicted in the case as partner in a fraud +scheme with Neidorf.) In Atlanta, the Prophet had already +pled guilty to one charge of conspiracy, one charge of wire fraud +and one charge of interstate transportation of stolen property. +The wire fraud charge, and the stolen property charge, +were both directly based on the E911 Document. + +The twenty-year-old Prophet proved a sorry customer, +answering questions politely but in a barely audible mumble, +his voice trailing off at the ends of sentences. +He was constantly urged to speak up. + +Cook, examining Prophet, forced him to admit that +he had once had a "drug problem," abusing amphetamines, +marijuana, cocaine, and LSD. This may have established +to the jury that "hackers" are, or can be, seedy lowlife characters, +but it may have damaged Prophet's credibility somewhat. +Zenner later suggested that drugs might have damaged Prophet's memory. +The interesting fact also surfaced that Prophet had never +physically met Craig Neidorf. He didn't even know +Neidorf's last name--at least, not until the trial. + +Prophet confirmed the basic facts of his hacker career. +He was a member of the Legion of Doom. He had abused codes, +he had broken into switching stations and re-routed calls, +he had hung out on pirate bulletin boards. He had raided +the BellSouth AIMSX computer, copied the E911 Document, +stored it on Jolnet, mailed it to Neidorf. He and Neidorf +had edited it, and Neidorf had known where it came from. + +Zenner, however, had Prophet confirm that Neidorf was not a member +of the Legion of Doom, and had not urged Prophet to break into +BellSouth computers. Neidorf had never urged Prophet to defraud anyone, +or to steal anything. Prophet also admitted that he had never known Neidorf +to break in to any computer. Prophet said that no one in the Legion of Doom +considered Craig Neidorf a "hacker" at all. Neidorf was not a UNIX maven, +and simply lacked the necessary skill and ability to break into computers. +Neidorf just published a magazine. + +On Friday, July 27, 1990, the case against Neidorf collapsed. +Cook moved to dismiss the indictment, citing "information currently +available to us that was not available to us at the inception of the trial." +Judge Bua praised the prosecution for this action, which he described as +"very responsible," then dismissed a juror and declared a mistrial. + +Neidorf was a free man. His defense, however, had cost himself +and his family dearly. Months of his life had been consumed in anguish; +he had seen his closest friends shun him as a federal criminal. +He owed his lawyers over a hundred thousand dollars, despite +a generous payment to the defense by Mitch Kapor. + +Neidorf was not found innocent. The trial was simply dropped. +Nevertheless, on September 9, 1991, Judge Bua granted Neidorf's +motion for the "expungement and sealing" of his indictment record. +The United States Secret Service was ordered to delete and destroy +all fingerprints, photographs, and other records of arrest +or processing relating to Neidorf's indictment, including +their paper documents and their computer records. + +Neidorf went back to school, blazingly determined to become a lawyer. +Having seen the justice system at work, Neidorf lost much of his enthusiasm +for merely technical power. At this writing, Craig Neidorf is working +in Washington as a salaried researcher for the American Civil Liberties Union. + +# + +The outcome of the Neidorf trial changed the EFF +from voices-in-the-wilderness to the media darlings +of the new frontier. + +Legally speaking, the Neidorf case was not a sweeping triumph +for anyone concerned. No constitutional principles had been established. +The issues of "freedom of the press" for electronic publishers remained +in legal limbo. There were public misconceptions about the case. +Many people thought Neidorf had been found innocent and relieved +of all his legal debts by Kapor. The truth was that the government +had simply dropped the case, and Neidorf's family had gone deeply +into hock to support him. + +But the Neidorf case did provide a single, devastating, public sound-bite: +THE FEDS SAID IT WAS WORTH EIGHTY GRAND, AND IT WAS ONLY WORTH THIRTEEN BUCKS. + +This is the Neidorf case's single most memorable element. No serious report +of the case missed this particular element. Even cops could not read this +without a wince and a shake of the head. It left the public credibility +of the crackdown agents in tatters. + +The crackdown, in fact, continued, however. Those two charges +against Prophet, which had been based on the E911 Document, +were quietly forgotten at his sentencing--even though Prophet +had already pled guilty to them. Georgia federal prosecutors +strongly argued for jail time for the Atlanta Three, insisting on +"the need to send a message to the community," "the message that +hackers around the country need to hear." + +There was a great deal in their sentencing memorandum +about the awful things that various other hackers had done +(though the Atlanta Three themselves had not, in fact, +actually committed these crimes). There was also much +speculation about the awful things that the Atlanta Three +MIGHT have done and WERE CAPABLE of doing (even though +they had not, in fact, actually done them). +The prosecution's argument carried the day. +The Atlanta Three were sent to prison: +Urvile and Leftist both got 14 months each, +while Prophet (a second offender) got 21 months. + +The Atlanta Three were also assessed staggering fines as "restitution": +$233,000 each. BellSouth claimed that the defendants had "stolen" +"approximately $233,880 worth" of "proprietary computer access information"-- +specifically, $233,880 worth of computer passwords and connect addresses. +BellSouth's astonishing claim of the extreme value of its own computer +passwords and addresses was accepted at face value by the Georgia court. +Furthermore (as if to emphasize its theoretical nature) this enormous sum +was not divvied up among the Atlanta Three, but each of them had to pay +all of it. + +A striking aspect of the sentence was that the Atlanta Three were +specifically forbidden to use computers, except for work or under supervision. +Depriving hackers of home computers and modems makes some sense if one +considers hackers as "computer addicts," but EFF, filing an amicus brief +in the case, protested that this punishment was unconstitutional-- +it deprived the Atlanta Three of their rights of free association +and free expression through electronic media. + +Terminus, the "ultimate hacker," was finally sent to prison for a year +through the dogged efforts of the Chicago Task Force. His crime, +to which he pled guilty, was the transfer of the UNIX password trapper, +which was officially valued by AT&T at $77,000, a figure which aroused +intense skepticism among those familiar with UNIX "login.c" programs. + +The jailing of Terminus and the Atlanta Legionnaires of Doom, however, +did not cause the EFF any sense of embarrassment or defeat. +On the contrary, the civil libertarians were rapidly gathering strength. + +An early and potent supporter was Senator Patrick Leahy, +Democrat from Vermont, who had been a Senate sponsor +of the Electronic Communications Privacy Act. Even before +the Neidorf trial, Leahy had spoken out in defense of hacker-power +and freedom of the keyboard: "We cannot unduly inhibit the inquisitive +13-year-old who, if left to experiment today, may tomorrow develop +the telecommunications or computer technology to lead the United States +into the 21st century. He represents our future and our best hope +to remain a technologically competitive nation." + +It was a handsome statement, rendered perhaps rather more effective +by the fact that the crackdown raiders DID NOT HAVE any Senators +speaking out for THEM. On the contrary, their highly secretive +actions and tactics, all "sealed search warrants" here and +"confidential ongoing investigations" there, might have won +them a burst of glamorous publicity at first, but were crippling +them in the on-going propaganda war. Gail Thackeray was reduced +to unsupported bluster: "Some of these people who are loudest +on the bandwagon may just slink into the background," +she predicted in Newsweek--when all the facts came out, +and the cops were vindicated. + +But all the facts did not come out. Those facts that did, +were not very flattering. And the cops were not vindicated. +And Gail Thackeray lost her job. By the end of 1991, +William Cook had also left public employment. + +1990 had belonged to the crackdown, but by '91 its agents +were in severe disarray, and the libertarians were on a roll. +People were flocking to the cause. + +A particularly interesting ally had been Mike Godwin of Austin, Texas. +Godwin was an individual almost as difficult to describe as Barlow; +he had been editor of the student newspaper of the University of Texas, +and a computer salesman, and a programmer, and in 1990 was back +in law school, looking for a law degree. + +Godwin was also a bulletin board maven. He was very well-known +in the Austin board community under his handle "Johnny Mnemonic," +which he adopted from a cyberpunk science fiction story by William Gibson. +Godwin was an ardent cyberpunk science fiction fan. As a fellow Austinite +of similar age and similar interests, I myself had known Godwin socially +for many years. When William Gibson and myself had been writing our +collaborative SF novel, The Difference Engine, Godwin had been our +technical advisor in our effort to link our Apple word-processors +from Austin to Vancouver. Gibson and I were so pleased by his generous +expert help that we named a character in the novel "Michael Godwin" +in his honor. + +The handle "Mnemonic" suited Godwin very well. His erudition +and his mastery of trivia were impressive to the point of stupor; +his ardent curiosity seemed insatiable, and his desire to debate +and argue seemed the central drive of his life. Godwin had even +started his own Austin debating society, wryly known as the +"Dull Men's Club." In person, Godwin could be overwhelming; +a flypaper-brained polymath who could not seem to let any idea go. +On bulletin boards, however, Godwin's closely reasoned, +highly grammatical, erudite posts suited the medium well, +and he became a local board celebrity. + +Mike Godwin was the man most responsible for the public national exposure +of the Steve Jackson case. The Izenberg seizure in Austin had received +no press coverage at all. The March 1 raids on Mentor, Bloodaxe, and +Steve Jackson Games had received a brief front-page splash in the +front page of the Austin American-Statesman, but it was confused +and ill-informed: the warrants were sealed, and the Secret Service +wasn't talking. Steve Jackson seemed doomed to obscurity. +Jackson had not been arrested; he was not charged with any crime; +he was not on trial. He had lost some computers in an ongoing +investigation--so what? Jackson tried hard to attract attention +to the true extent of his plight, but he was drawing a blank; +no one in a position to help him seemed able to get a mental grip +on the issues. + +Godwin, however, was uniquely, almost magically, qualified +to carry Jackson's case to the outside world. Godwin was +a board enthusiast, a science fiction fan, a former journalist, +a computer salesman, a lawyer-to-be, and an Austinite. +Through a coincidence yet more amazing, in his last year +of law school Godwin had specialized in federal prosecutions +and criminal procedure. Acting entirely on his own, Godwin made +up a press packet which summarized the issues and provided useful +contacts for reporters. Godwin's behind-the-scenes effort +(which he carried out mostly to prove a point in a local board debate) +broke the story again in the Austin American-Statesman and then in Newsweek. + +Life was never the same for Mike Godwin after that. As he joined the growing +civil liberties debate on the Internet, it was obvious to all parties involved +that here was one guy who, in the midst of complete murk and confusion, +GENUINELY UNDERSTOOD EVERYTHING HE WAS TALKING ABOUT. The disparate elements +of Godwin's dilettantish existence suddenly fell together as neatly as +the facets of a Rubik's cube. + +When the time came to hire a full-time EFF staff attorney, +Godwin was the obvious choice. He took the Texas bar exam, +left Austin, moved to Cambridge, became a full-time, professional, +computer civil libertarian, and was soon touring the nation on behalf +of EFF, delivering well-received addresses on the issues to crowds +as disparate as academics, industrialists, science fiction fans, +and federal cops. + +Michael Godwin is currently the chief legal counsel of +the Electronic Frontier Foundation in Cambridge, Massachusetts. + +# + +Another early and influential participant in the controversy +was Dorothy Denning. Dr. Denning was unique among investigators +of the computer underground in that she did not enter the debate +with any set of politicized motives. She was a professional +cryptographer and computer security expert whose primary interest +in hackers was SCHOLARLY. She had a B.A. and M.A. in mathematics, +and a Ph.D. in computer science from Purdue. She had worked for SRI +International, the California think-tank that was also the home of +computer-security maven Donn Parker, and had authored an influential text +called Cryptography and Data Security. In 1990, Dr. Denning was working for +Digital Equipment Corporation in their Systems Reseach Center. Her husband, +Peter Denning, was also a computer security expert, working for NASA's +Research Institute for Advanced Computer Science. He had edited the +well-received Computers Under Attack: Intruders, Worms and Viruses. + +Dr. Denning took it upon herself to contact the digital underground, +more or less with an anthropological interest. There she discovered +that these computer-intruding hackers, who had been characterized +as unethical, irresponsible, and a serious danger to society, +did in fact have their own subculture and their own rules. +They were not particularly well-considered rules, but they were, +in fact, rules. Basically, they didn't take money and they +didn't break anything. + +Her dispassionate reports on her researches did a great deal +to influence serious-minded computer professionals--the sort +of people who merely rolled their eyes at the cyberspace +rhapsodies of a John Perry Barlow. + +For young hackers of the digital underground, meeting Dorothy Denning +was a genuinely mind-boggling experience. Here was this neatly coiffed, +conservatively dressed, dainty little personage, who reminded most +hackers of their moms or their aunts. And yet she was an IBM systems +programmer with profound expertise in computer architectures +and high-security information flow, who had personal friends +in the FBI and the National Security Agency. + +Dorothy Denning was a shining example of the American mathematical +intelligentsia, a genuinely brilliant person from the central ranks +of the computer-science elite. And here she was, gently questioning +twenty-year-old hairy-eyed phone-phreaks over the deeper ethical +implications of their behavior. + +Confronted by this genuinely nice lady, most hackers sat up very straight +and did their best to keep the anarchy-file stuff down to a faint whiff +of brimstone. Nevertheless, the hackers WERE in fact prepared to seriously +discuss serious issues with Dorothy Denning. They were willing to speak +the unspeakable and defend the indefensible, to blurt out their convictions +that information cannot be owned, that the databases of governments and large +corporations were a threat to the rights and privacy of individuals. + +Denning's articles made it clear to many that "hacking" +was not simple vandalism by some evil clique of psychotics. +"Hacking" was not an aberrant menace that could be charmed away +by ignoring it, or swept out of existence by jailing a few ringleaders. +Instead, "hacking" was symptomatic of a growing, primal struggle over +knowledge and power in the age of information. + +Denning pointed out that the attitude of hackers were at least partially +shared by forward-looking management theorists in the business community: +people like Peter Drucker and Tom Peters. Peter Drucker, in his book +The New Realities, had stated that "control of information by the government +is no longer possible. Indeed, information is now transnational. +Like money, it has no `fatherland.'" + +And management maven Tom Peters had chided large corporations for uptight, +proprietary attitudes in his bestseller, Thriving on Chaos: +"Information hoarding, especially by politically motivated, +power-seeking staffs, had been commonplace throughout American industry, +service and manufacturing alike. It will be an impossible +millstone aroung the neck of tomorrow's organizations." + +Dorothy Denning had shattered the social membrane of the +digital underground. She attended the Neidorf trial, +where she was prepared to testify for the defense as an expert witness. +She was a behind-the-scenes organizer of two of the most important +national meetings of the computer civil libertarians. Though not +a zealot of any description, she brought disparate elements of the +electronic community into a surprising and fruitful collusion. + +Dorothy Denning is currently the Chair of the Computer Science Department +at Georgetown University in Washington, DC. + +# + +There were many stellar figures in the civil libertarian community. +There's no question, however, that its single most influential figure +was Mitchell D. Kapor. Other people might have formal titles, +or governmental positions, have more experience with crime, +or with the law, or with the arcanities of computer security +or constitutional theory. But by 1991 Kapor had transcended +any such narrow role. Kapor had become "Mitch." + +Mitch had become the central civil-libertarian ad-hocrat. +Mitch had stood up first, he had spoken out loudly, directly, +vigorously and angrily, he had put his own reputation, +and his very considerable personal fortune, on the line. +By mid-'91 Kapor was the best-known advocate of his cause +and was known PERSONALLY by almost every single human being in America +with any direct influence on the question of civil liberties in cyberspace. +Mitch had built bridges, crossed voids, changed paradigms, forged metaphors, +made phone-calls and swapped business cards to such spectacular effect +that it had become impossible for anyone to take any action in the +"hacker question" without wondering what Mitch might think-- +and say--and tell his friends. + +The EFF had simply NETWORKED the situation into an entirely new status quo. +And in fact this had been EFF's deliberate strategy from the beginning. +Both Barlow and Kapor loathed bureaucracies and had deliberately +chosen to work almost entirely through the electronic spiderweb of +"valuable personal contacts." + +After a year of EFF, both Barlow and Kapor had every reason +to look back with satisfaction. EFF had established its own Internet node, +"eff.org," with a well-stocked electronic archive of documents on +electronic civil rights, privacy issues, and academic freedom. +EFF was also publishing EFFector, a quarterly printed journal, +as well as EFFector Online, an electronic newsletter with +over 1,200 subscribers. And EFF was thriving on the Well. + +EFF had a national headquarters in Cambridge and a full-time staff. +It had become a membership organization and was attracting +grass-roots support. It had also attracted the support +of some thirty civil-rights lawyers, ready and eager +to do pro bono work in defense of the Constitution in Cyberspace. + +EFF had lobbied successfully in Washington and in Massachusetts +to change state and federal legislation on computer networking. +Kapor in particular had become a veteran expert witness, +and had joined the Computer Science and Telecommunications Board +of the National Academy of Science and Engineering. + +EFF had sponsored meetings such as "Computers, Freedom and Privacy" +and the CPSR Roundtable. It had carried out a press offensive that, +in the words of EFFector, "has affected the climate of opinion about +computer networking and begun to reverse the slide into +`hacker hysteria' that was beginning to grip the nation." + +It had helped Craig Neidorf avoid prison. + +And, last but certainly not least, the Electronic Frontier Foundation +had filed a federal lawsuit in the name of Steve Jackson, +Steve Jackson Games Inc., and three users of the Illuminati +bulletin board system. The defendants were, and are, +the United States Secret Service, William Cook, Tim Foley, +Barbara Golden and Henry Kleupfel. + +The case, which is in pre-trial procedures in an Austin federal court +as of this writing, is a civil action for damages to redress +alleged violations of the First and Fourth Amendments to the +United States Constitution, as well as the Privacy Protection Act +of 1980 (42 USC 2000aa et seq.), and the Electronic Communications +Privacy Act (18 USC 2510 et seq and 2701 et seq). + +EFF had established that it had credibility. It had also established +that it had teeth. + +In the fall of 1991 I travelled to Massachusetts to speak personally +with Mitch Kapor. It was my final interview for this book. + +# + +The city of Boston has always been one of the major intellectual centers +of the American republic. It is a very old city by American standards, +a place of skyscrapers overshadowing seventeenth-century graveyards, +where the high-tech start-up companies of Route 128 co-exist with the +hand-wrought pre-industrial grace of "Old Ironsides," the USS CONSTITUTION. + +The Battle of Bunker Hill, one of the first and bitterest armed clashes +of the American Revolution, was fought in Boston's environs. Today there is +a monumental spire on Bunker Hill, visible throughout much of the city. +The willingness of the republican revolutionaries to take up arms and fire +on their oppressors has left a cultural legacy that two full centuries +have not effaced. Bunker Hill is still a potent center of American political +symbolism, and the Spirit of '76 is still a potent image for those who seek +to mold public opinion. + +Of course, not everyone who wraps himself in the flag is necessarily +a patriot. When I visited the spire in September 1991, it bore a huge, +badly-erased, spray-can grafitto around its bottom reading +"BRITS OUT--IRA PROVOS." Inside this hallowed edifice was +a glass-cased diorama of thousands of tiny toy soldiers, +rebels and redcoats, fighting and dying over the green hill, +the riverside marshes, the rebel trenchworks. Plaques indicated the +movement of troops, the shiftings of strategy. The Bunker Hill Monument +is occupied at its very center by the toy soldiers of a military +war-game simulation. + +The Boston metroplex is a place of great universities, +prominent among the Massachusetts Institute of Technology, +where the term "computer hacker" was first coined. The Hacker Crackdown +of 1990 might be interpreted as a political struggle among American cities: +traditional strongholds of longhair intellectual liberalism, +such as Boston, San Francisco, and Austin, versus the bare-knuckle +industrial pragmatism of Chicago and Phoenix (with Atlanta and New York +wrapped in internal struggle). + +The headquarters of the Electronic Frontier Foundation is on +155 Second Street in Cambridge, a Bostonian suburb north +of the River Charles. Second Street has weedy sidewalks of dented, +sagging brick and elderly cracked asphalt; large street-signs warn +"NO PARKING DURING DECLARED SNOW EMERGENCY." This is an old area +of modest manufacturing industries; the EFF is catecorner from the +Greene Rubber Company. EFF's building is two stories of red brick; +its large wooden windows feature gracefully arched tops and stone sills. + +The glass window beside the Second Street entrance bears three sheets +of neatly laser-printed paper, taped against the glass. They read: +ON Technology. EFF. KEI. + +"ON Technology" is Kapor's software company, which currently specializes +in "groupware" for the Apple Macintosh computer. "Groupware" is intended +to promote efficient social interaction among office-workers linked +by computers. ON Technology's most successful software products to date +are "Meeting Maker" and "Instant Update." + +"KEI" is Kapor Enterprises Inc., Kapor's personal holding company, +the commercial entity that formally controls his extensive investments +in other hardware and software corporations. + +"EFF" is a political action group--of a special sort. + +Inside, someone's bike has been chained to the handrails +of a modest flight of stairs. A wall of modish glass brick +separates this anteroom from the offices. Beyond the brick, +there's an alarm system mounted on the wall, a sleek, complex little +number that resembles a cross between a thermostat and a CD player. +Piled against the wall are box after box of a recent special issue +of Scientific American, "How to Work, Play, and Thrive in Cyberspace," +with extensive coverage of electronic networking techniques +and political issues, including an article by Kapor himself. +These boxes are addressed to Gerard Van der Leun, EFF's +Director of Communications, who will shortly mail those magazines +to every member of the EFF. + +The joint headquarters of EFF, KEI, and ON Technology, +which Kapor currently rents, is a modestly bustling place. +It's very much the same physical size as Steve Jackson's gaming company. +It's certainly a far cry from the gigantic gray steel-sided railway +shipping barn, on the Monsignor O'Brien Highway, that is owned +by Lotus Development Corporation. + +Lotus is, of course, the software giant that Mitchell Kapor founded +in the late 70s. The software program Kapor co-authored, +"Lotus 1-2-3," is still that company's most profitable product. +"Lotus 1-2-3" also bears a singular distinction in the +digital underground: it's probably the most pirated piece +of application software in world history. + +Kapor greets me cordially in his own office, down a hall. +Kapor, whose name is pronounced KAY-por, is in his early forties, +married and the father of two. He has a round face, high forehead, +straight nose, a slightly tousled mop of black hair peppered with gray. +His large brown eyes are wideset, reflective, one might almost say soulful. +He disdains ties, and commonly wears Hawaiian shirts and tropical prints, +not so much garish as simply cheerful and just that little bit anomalous. + +There is just the whiff of hacker brimstone about Mitch Kapor. +He may not have the hard-riding, hell-for-leather, guitar-strumming +charisma of his Wyoming colleague John Perry Barlow, but there's +something about the guy that still stops one short. He has the air +of the Eastern city dude in the bowler hat, the dreamy, +Longfellow-quoting poker shark who only HAPPENS to know +the exact mathematical odds against drawing to an inside straight. +Even among his computer-community colleagues, who are hardly known +for mental sluggishness, Kapor strikes one forcefully as a very +intelligent man. He speaks rapidly, with vigorous gestures, +his Boston accent sometimes slipping to the sharp nasal tang +of his youth in Long Island. + +Kapor, whose Kapor Family Foundation does much of his philanthropic work, +is a strong supporter of Boston's Computer Museum. Kapor's interest +in the history of his industry has brought him some remarkable curios, +such as the "byte" just outside his office door. This "byte"-- +eight digital bits--has been salvaged from the wreck of an +electronic computer of the pre-transistor age. It's a standing gunmetal +rack about the size of a small toaster-oven: with eight slots +of hand-soldered breadboarding featuring thumb-sized vacuum tubes. +If it fell off a table it could easily break your foot, +but it was state-of-the-art computation in the 1940s. +(It would take exactly 157,184 of these primordial toasters +to hold the first part of this book.) + +There's also a coiling, multicolored, scaly dragon that some +inspired techno-punk artist has cobbled up entirely out of transistors, +capacitors, and brightly plastic-coated wiring. + +Inside the office, Kapor excuses himself briefly to do a little +mouse-whizzing housekeeping on his personal Macintosh IIfx. +If its giant screen were an open window, an agile person +could climb through it without much trouble at all. +There's a coffee-cup at Kapor's elbow, a memento of his +recent trip to Eastern Europe, which has a black-and-white +stencilled photo and the legend CAPITALIST FOOLS TOUR. +It's Kapor, Barlow, and two California venture-capitalist luminaries +of their acquaintance, four windblown, grinning Baby Boomer +dudes in leather jackets, boots, denim, travel bags, +standing on airport tarmac somewhere behind the formerly Iron Curtain. +They look as if they're having the absolute time of their lives. + +Kapor is in a reminiscent mood. We talk a bit about his youth-- +high school days as a "math nerd," Saturdays attending Columbia University's +high-school science honors program, where he had his first experience +programming computers. IBM 1620s, in 1965 and '66. "I was very interested," +says Kapor, "and then I went off to college and got distracted by drugs sex +and rock and roll, like anybody with half a brain would have then!" +After college he was a progressive-rock DJ in Hartford, Connecticut, +for a couple of years. + +I ask him if he ever misses his rock and roll days--if he ever wished +he could go back to radio work. + +He shakes his head flatly. "I stopped thinking about going back +to be a DJ the day after Altamont." + +Kapor moved to Boston in 1974 and got a job programming mainframes in COBOL. +He hated it. He quit and became a teacher of transcendental meditation. +(It was Kapor's long flirtation with Eastern mysticism that gave the +world "Lotus.") + +In 1976 Kapor went to Switzerland, where the Transcendental Meditation +movement had rented a gigantic Victorian hotel in St-Moritz. It was +an all-male group--a hundred and twenty of them--determined upon +Enlightenment or Bust. Kapor had given the transcendant his best shot. +He was becoming disenchanted by "the nuttiness in the organization." +"They were teaching people to levitate," he says, staring at the floor. +His voice drops an octave, becomes flat. "THEY DON'T LEVITATE." + +Kapor chose Bust. He went back to the States and acquired a degree +in counselling psychology. He worked a while in a hospital, +couldn't stand that either. "My rep was," he says "a very bright kid +with a lot of potential who hasn't found himself. Almost thirty. +Sort of lost." + +Kapor was unemployed when he bought his first personal computer--an Apple II. +He sold his stereo to raise cash and drove to New Hampshire to avoid the +sales tax. + +"The day after I purchased it," Kapor tells me, "I was hanging out +in a computer store and I saw another guy, a man in his forties, +well-dressed guy, and eavesdropped on his conversation with the salesman. +He didn't know anything about computers. I'd had a year programming. +And I could program in BASIC. I'd taught myself. So I went up to him, +and I actually sold myself to him as a consultant." He pauses. +"I don't know where I got the nerve to do this. It was uncharacteristic. +I just said, `I think I can help you, I've been listening, +this is what you need to do and I think I can do it for you.' +And he took me on! He was my first client! I became a computer +consultant the first day after I bought the Apple II." + +Kapor had found his true vocation. He attracted more clients +for his consultant service, and started an Apple users' group. + +A friend of Kapor's, Eric Rosenfeld, a graduate student at MIT, +had a problem. He was doing a thesis on an arcane form of +financial statistics, but could not wedge himself into the crowded queue +for time on MIT's mainframes. (One might note at this point that if +Mr. Rosenfeld had dishonestly broken into the MIT mainframes, +Kapor himself might have never invented Lotus 1-2-3 and +the PC business might have been set back for years!) +Eric Rosenfeld did have an Apple II, however, +and he thought it might be possible to scale the problem down. +Kapor, as favor, wrote a program for him in BASIC that did the job. + +It then occurred to the two of them, out of the blue, +that it might be possible to SELL this program. +They marketed it themselves, in plastic baggies, +for about a hundred bucks a pop, mail order. +"This was a total cottage industry by a marginal consultant," +Kapor says proudly. "That's how I got started, honest to God." + +Rosenfeld, who later became a very prominent figure on Wall Street, +urged Kapor to go to MIT's business school for an MBA. +Kapor did seven months there, but never got his MBA. +He picked up some useful tools--mainly a firm grasp +of the principles of accounting--and, in his own words, +"learned to talk MBA." Then he dropped out and went to Silicon Valley. + +The inventors of VisiCalc, the Apple computer's premier business program, +had shown an interest in Mitch Kapor. Kapor worked diligently for them +for six months, got tired of California, and went back to Boston +where they had better bookstores. The VisiCalc group had made +the critical error of bringing in "professional management." +"That drove them into the ground," Kapor says. + +"Yeah, you don't hear a lot about VisiCalc these days," I muse. + +Kapor looks surprised. "Well, Lotus. . . we BOUGHT it." + +"Oh. You BOUGHT it?" + +"Yeah." + +"Sort of like the Bell System buying Western Union?" + +Kapor grins. "Yep! Yep! Yeah, exactly!" + +Mitch Kapor was not in full command of the destiny of himself +or his industry. The hottest software commodities of the early 1980s +were COMPUTER GAMES--the Atari seemed destined to enter every teenage home +in America. Kapor got into business software simply because he didn't have +any particular feeling for computer games. But he was supremely fast +on his feet, open to new ideas and inclined to trust his instincts. +And his instincts were good. He chose good people to deal with-- +gifted programmer Jonathan Sachs (the co-author of Lotus 1-2-3). +Financial wizard Eric Rosenfeld, canny Wall Street analyst +and venture capitalist Ben Rosen. Kapor was the founder and CEO of Lotus, +one of the most spectacularly successful business ventures of the +later twentieth century. + +He is now an extremely wealthy man. I ask him if he actually +knows how much money he has. + +"Yeah," he says. "Within a percent or two." + +How much does he actually have, then? + +He shakes his head. "A lot. A lot. Not something I talk about. +Issues of money and class are things that cut pretty close to the bone." + +I don't pry. It's beside the point. One might presume, impolitely, +that Kapor has at least forty million--that's what he got the year +he left Lotus. People who ought to know claim Kapor has about +a hundred and fifty million, give or take a market swing +in his stock holdings. If Kapor had stuck with Lotus, +as his colleague friend and rival Bill Gates has stuck +with his own software start-up, Microsoft, then Kapor +would likely have much the same fortune Gates has-- +somewhere in the neighborhood of three billion, +give or take a few hundred million. Mitch Kapor +has all the money he wants. Money has lost whatever charm +it ever held for him--probably not much in the first place. +When Lotus became too uptight, too bureaucratic, too far +from the true sources of his own satisfaction, Kapor walked. +He simply severed all connections with the company and went out the door. +It stunned everyone--except those who knew him best. + +Kapor has not had to strain his resources to wreak a thorough +transformation in cyberspace politics. In its first year, +EFF's budget was about a quarter of a million dollars. +Kapor is running EFF out of his pocket change. + +Kapor takes pains to tell me that he does not consider himself +a civil libertarian per se. He has spent quite some time +with true-blue civil libertarians lately, and there's a +political-correctness to them that bugs him. They seem +to him to spend entirely too much time in legal nitpicking +and not enough vigorously exercising civil rights in the +everyday real world. + +Kapor is an entrepreneur. Like all hackers, he prefers his involvements +direct, personal, and hands-on. "The fact that EFF has a node on the +Internet is a great thing. We're a publisher. We're a distributor +of information." Among the items the eff.org Internet node carries +is back issues of Phrack. They had an internal debate about that in EFF, +and finally decided to take the plunge. They might carry other +digital underground publications--but if they do, he says, +"we'll certainly carry Donn Parker, and anything Gail Thackeray +wants to put up. We'll turn it into a public library, that has +the whole spectrum of use. Evolve in the direction of people making up +their own minds." He grins. "We'll try to label all the editorials." + +Kapor is determined to tackle the technicalities of the Internet +in the service of the public interest. "The problem with being a node +on the Net today is that you've got to have a captive technical specialist. +We have Chris Davis around, for the care and feeding of the balky beast! +We couldn't do it ourselves!" + +He pauses. "So one direction in which technology has to evolve +is much more standardized units, that a non-technical person +can feel comfortable with. It's the same shift as from minicomputers to PCs. +I can see a future in which any person can have a Node on the Net. +Any person can be a publisher. It's better than the media we now have. +It's possible. We're working actively." + +Kapor is in his element now, fluent, thoroughly in command in his material. +"You go tell a hardware Internet hacker that everyone should have a node +on the Net," he says, "and the first thing they're going to say is, +`IP doesn't scale!'" ("IP" is the interface protocol for the Internet. +As it currently exists, the IP software is simply not capable of +indefinite expansion; it will run out of usable addresses, it will saturate.) +"The answer," Kapor says, "is: evolve the protocol! Get the smart people +together and figure out what to do. Do we add ID? Do we add new protocol? +Don't just say, WE CAN'T DO IT." + +Getting smart people together to figure out what to do is a skill +at which Kapor clearly excels. I counter that people on the Internet +rather enjoy their elite technical status, and don't seem particularly +anxious to democratize the Net. + +Kapor agrees, with a show of scorn. "I tell them that this is the snobbery +of the people on the Mayflower looking down their noses at the people +who came over ON THE SECOND BOAT! Just because they got here a year, +or five years, or ten years before everybody else, that doesn't give +them ownership of cyberspace! By what right?" + +I remark that the telcos are an electronic network, too, +and they seem to guard their specialized knowledge pretty closely. + +Kapor ripostes that the telcos and the Internet are entirely +different animals. "The Internet is an open system, +everything is published, everything gets argued about, +basically by anybody who can get in. Mostly, it's exclusive +and elitist just because it's so difficult. Let's make it easier to use." + +On the other hand, he allows with a swift change of emphasis, +the so-called elitists do have a point as well. "Before people start coming in, +who are new, who want to make suggestions, and criticize the Net as +`all screwed up'. . . . They should at least take the time to understand +the culture on its own terms. It has its own history--show some respect +for it. I'm a conservative, to that extent." + +The Internet is Kapor's paradigm for the future of telecommunications. +The Internet is decentralized, non-hierarchical, almost anarchic. +There are no bosses, no chain of command, no secret data. +If each node obeys the general interface standards, +there's simply no need for any central network authority. + +Wouldn't that spell the doom of AT&T as an institution? I ask. + +That prospect doesn't faze Kapor for a moment. "Their big advantage, +that they have now, is that they have all of the wiring. +But two things are happening. Anyone with right-of-way +is putting down fiber--Southern Pacific Railroad, +people like that--there's enormous `dark fiber' laid in." +("Dark Fiber" is fiber-optic cable, whose enormous capacity +so exceeds the demands of current usage that much of the +fiber still has no light-signals on it--it's still `dark,' +awaiting future use.) + +"The other thing that's happening is the local-loop stuff +is going to go wireless. Everyone from Bellcore to the cable TV +companies to AT&T wants to put in these things called +`personal communication systems.' So you could have local competition-- +you could have multiplicity of people, a bunch of neighborhoods, +sticking stuff up on poles. And a bunch of other people laying in dark fiber. +So what happens to the telephone companies? There's enormous pressure +on them from both sides. + +"The more I look at this, the more I believe that in a post-industrial, +digital world, the idea of regulated monopolies is bad. People will +look back on it and say that in the 19th and 20th centuries +the idea of public utilities was an okay compromise. +You needed one set of wires in the ground. It was too economically +inefficient, otherwise. And that meant one entity running it. +But now, with pieces being wireless--the connections are going +to be via high-level interfaces, not via wires. I mean, ULTIMATELY +there are going to be wires--but the wires are just a commodity. +Fiber, wireless. You no longer NEED a utility." + +Water utilities? Gas utilities? + +Of course we still need those, he agrees. "But when what you're moving +is information, instead of physical substances, then you can play by +a different set of rules. We're evolving those rules now! +Hopefully you can have a much more decentralized system, +and one in which there's more competition in the marketplace. + +"The role of government will be to make sure that nobody cheats. +The proverbial `level playing field.' A policy that prevents monopolization. +It should result in better service, lower prices, more choices, +and local empowerment." He smiles. "I'm very big on local empowerment." + +Kapor is a man with a vision. It's a very novel vision which he +and his allies are working out in considerable detail and with great energy. +Dark, cynical, morbid cyberpunk that I am, I cannot avoid considering +some of the darker implications of "decentralized, nonhierarchical, +locally empowered" networking. + +I remark that some pundits have suggested that electronic networking--faxes, +phones, small-scale photocopiers--played a strong role in dissolving +the power of centralized communism and causing the collapse of the Warsaw Pact. + +Socialism is totally discredited, says Kapor, fresh back from +the Eastern Bloc. The idea that faxes did it, all by themselves, +is rather wishful thinking. + +Has it occurred to him that electronic networking might corrode +America's industrial and political infrastructure to the point +where the whole thing becomes untenable, unworkable--and the old order +just collapses headlong, like in Eastern Europe? + +"No," Kapor says flatly. "I think that's extraordinarily unlikely. +In part, because ten or fifteen years ago, I had similar hopes +about personal computers--which utterly failed to materialize." +He grins wryly, then his eyes narrow. "I'm VERY opposed to techno-utopias. +Every time I see one, I either run away, or try to kill it." + +It dawns on me then that Mitch Kapor is not trying to +make the world safe for democracy. He certainly is not +trying to make it safe for anarchists or utopians-- +least of all for computer intruders or electronic rip-off artists. +What he really hopes to do is make the world safe for +future Mitch Kapors. This world of decentralized, small-scale nodes, +with instant global access for the best and brightest, +would be a perfect milieu for the shoestring attic capitalism +that made Mitch Kapor what he is today. + +Kapor is a very bright man. He has a rare combination +of visionary intensity with a strong practical streak. +The Board of the EFF: John Barlow, Jerry Berman of the ACLU, +Stewart Brand, John Gilmore, Steve Wozniak, and Esther Dyson, +the doyenne of East-West computer entrepreneurism--share his gift, +his vision, and his formidable networking talents. +They are people of the 1960s, winnowed-out by its turbulence +and rewarded with wealth and influence. They are some of the best +and the brightest that the electronic community has to offer. +But can they do it, in the real world? Or are they only dreaming? +They are so few. And there is so much against them. + +I leave Kapor and his networking employees struggling cheerfully +with the promising intricacies of their newly installed Macintosh +System 7 software. The next day is Saturday. EFF is closed. +I pay a few visits to points of interest downtown. + +One of them is the birthplace of the telephone. + +It's marked by a bronze plaque in a plinth of black-and-white speckled granite. It sits in the +plaza of the John F. Kennedy Federal Building, the very place where Kapor was +once fingerprinted by the FBI. + +The plaque has a bas-relief picture of Bell's original telephone. +"BIRTHPLACE OF THE TELEPHONE," it reads. "Here, on June 2, 1875, +Alexander Graham Bell and Thomas A. Watson first transmitted sound over wires. + +"This successful experiment was completed in a fifth floor garret +at what was then 109 Court Street and marked the beginning of +world-wide telephone service." + +109 Court Street is long gone. Within sight of Bell's plaque, +across a street, is one of the central offices of NYNEX, +the local Bell RBOC, on 6 Bowdoin Square. + +I cross the street and circle the telco building, slowly, +hands in my jacket pockets. It's a bright, windy, New England +autumn day. The central office is a handsome 1940s-era megalith +in late Art Deco, eight stories high. + +Parked outside the back is a power-generation truck. +The generator strikes me as rather anomalous. Don't they +already have their own generators in this eight-story monster? +Then the suspicion strikes me that NYNEX must have heard +of the September 17 AT&T power-outage which crashed New York City. +Belt-and-suspenders, this generator. Very telco. + +Over the glass doors of the front entrance is a handsome bronze +bas-relief of Art Deco vines, sunflowers, and birds, entwining +the Bell logo and the legend NEW ENGLAND TELEPHONE AND TELEGRAPH COMPANY +--an entity which no longer officially exists. + +The doors are locked securely. I peer through the shadowed glass. +Inside is an official poster reading: + + +"New England Telephone a NYNEX Company + +ATTENTION + +"All persons while on New England Telephone +Company premises are required to visibly wear their +identification cards (C.C.P. Section 2, Page 1). + +"Visitors, vendors, contractors, and all others are +required to visibly wear a daily pass. + +"Thank you. + +Kevin C. Stanton. +Building Security Coordinator." + + +Outside, around the corner, is a pull-down ribbed metal security door, +a locked delivery entrance. Some passing stranger has grafitti-tagged +this door, with a single word in red spray-painted cursive: + +Fury + +# + +My book on the Hacker Crackdown is almost over now. +I have deliberately saved the best for last. + +In February 1991, I attended the CPSR Public Policy Roundtable, +in Washington, DC. CPSR, Computer Professionals for Social Responsibility, +was a sister organization of EFF, or perhaps its aunt, being older +and perhaps somewhat wiser in the ways of the world of politics. + +Computer Professionals for Social Responsibility began in 1981 +in Palo Alto, as an informal discussion group of Californian +computer scientists and technicians, united by nothing more +than an electronic mailing list. This typical high-tech +ad-hocracy received the dignity of its own acronym in 1982, +and was formally incorporated in 1983. + +CPSR lobbied government and public alike with an educational +outreach effort, sternly warning against any foolish +and unthinking trust in complex computer systems. +CPSR insisted that mere computers should never be +considered a magic panacea for humanity's social, +ethical or political problems. CPSR members were especially +troubled about the stability, safety, and dependability +of military computer systems, and very especially troubled +by those systems controlling nuclear arsenals. CPSR was +best-known for its persistent and well-publicized attacks on the +scientific credibility of the Strategic Defense Initiative ("Star Wars"). + +In 1990, CPSR was the nation's veteran cyber-political activist group, +with over two thousand members in twenty- one local chapters across the US. +It was especially active in Boston, Silicon Valley, and Washington DC, +where its Washington office sponsored the Public Policy Roundtable. + +The Roundtable, however, had been funded by EFF, which had passed CPSR +an extensive grant for operations. This was the first large-scale, +official meeting of what was to become the electronic civil +libertarian community. + +Sixty people attended, myself included--in this instance, not so much +as a journalist as a cyberpunk author. Many of the luminaries +of the field took part: Kapor and Godwin as a matter of course. +Richard Civille and Marc Rotenberg of CPSR. Jerry Berman of the ACLU. +John Quarterman, author of The Matrix. Steven Levy, author of Hackers. +George Perry and Sandy Weiss of Prodigy Services, there to network +about the civil-liberties troubles their young commercial +network was experiencing. Dr. Dorothy Denning. Cliff Figallo, +manager of the Well. Steve Jackson was there, having finally +found his ideal target audience, and so was Craig Neidorf, +"Knight Lightning" himself, with his attorney, Sheldon Zenner. +Katie Hafner, science journalist, and co-author of Cyberpunk: +Outlaws and Hackers on the Computer Frontier. Dave Farber, +ARPAnet pioneer and fabled Internet guru. Janlori Goldman +of the ACLU's Project on Privacy and Technology. John Nagle +of Autodesk and the Well. Don Goldberg of the House Judiciary Committee. +Tom Guidoboni, the defense attorney in the Internet Worm case. +Lance Hoffman, computer-science professor at The George Washington +University. Eli Noam of Columbia. And a host of others no less distinguished. + +Senator Patrick Leahy delivered the keynote address, +expressing his determination to keep ahead of the curve +on the issue of electronic free speech. The address was +well-received, and the sense of excitement was palpable. +Every panel discussion was interesting--some were entirely +compelling. People networked with an almost frantic interest. + +I myself had a most interesting and cordial lunch discussion with +Noel and Jeanne Gayler, Admiral Gayler being a former director +of the National Security Agency. As this was the first known encounter +between an actual no-kidding cyberpunk and a chief executive of +America's largest and best-financed electronic espionage apparat, +there was naturally a bit of eyebrow-raising on both sides. + +Unfortunately, our discussion was off-the-record. In fact +all the discussions at the CPSR were officially off-the-record, +the idea being to do some serious networking in an atmosphere +of complete frankness, rather than to stage a media circus. + +In any case, CPSR Roundtable, though interesting and intensely valuable, +was as nothing compared to the truly mind-boggling event that transpired +a mere month later. + +# + +"Computers, Freedom and Privacy." Four hundred people from +every conceivable corner of America's electronic community. +As a science fiction writer, I have been to some weird gigs in my day, +but this thing is truly BEYOND THE PALE. Even "Cyberthon," +Point Foundation's "Woodstock of Cyberspace" where Bay Area +psychedelia collided headlong with the emergent world +of computerized virtual reality, was like a Kiwanis Club gig +compared to this astonishing do. + +The "electronic community" had reached an apogee. +Almost every principal in this book is in attendance. +Civil Libertarians. Computer Cops. The Digital Underground. +Even a few discreet telco people. Colorcoded dots +for lapel tags are distributed. Free Expression issues. +Law Enforcement. Computer Security. Privacy. Journalists. +Lawyers. Educators. Librarians. Programmers. +Stylish punk-black dots for the hackers and phone phreaks. +Almost everyone here seems to wear eight or nine dots, +to have six or seven professional hats. + +It is a community. Something like Lebanon perhaps, +but a digital nation. People who had feuded all year +in the national press, people who entertained the deepest +suspicions of one another's motives and ethics, are now +in each others' laps. "Computers, Freedom and Privacy" +had every reason in the world to turn ugly, and yet except +for small irruptions of puzzling nonsense from the +convention's token lunatic, a surprising bonhomie reigned. +CFP was like a wedding-party in which two lovers, +unstable bride and charlatan groom, tie the knot +in a clearly disastrous matrimony. + +It is clear to both families--even to neighbors and random guests-- +that this is not a workable relationship, and yet the young couple's +desperate attraction can brook no further delay. They simply cannot +help themselves. Crockery will fly, shrieks from their newlywed home +will wake the city block, divorce waits in the wings like a vulture +over the Kalahari, and yet this is a wedding, and there is going +to be a child from it. Tragedies end in death; comedies in marriage. +The Hacker Crackdown is ending in marriage. And there will be a child. + +From the beginning, anomalies reign. John Perry Barlow, +cyberspace ranger, is here. His color photo in +The New York Times Magazine, Barlow scowling +in a grim Wyoming snowscape, with long black coat, +dark hat, a Macintosh SE30 propped on a fencepost +and an awesome frontier rifle tucked under one arm, +will be the single most striking visual image +of the Hacker Crackdown. And he is CFP's guest of honor-- +along with Gail Thackeray of the FCIC! What on earth do +they expect these dual guests to do with each other? Waltz? + +Barlow delivers the first address. Uncharacteristically, +he is hoarse--the sheer volume of roadwork has worn him down. +He speaks briefly, congenially, in a plea for conciliation, +and takes his leave to a storm of applause. + +Then Gail Thackeray takes the stage. She's visibly nervous. +She's been on the Well a lot lately. Reading those Barlow posts. +Following Barlow is a challenge to anyone. In honor of the famous +lyricist for the Grateful Dead, she announces reedily, she is going to read-- +A POEM. A poem she has composed herself. + +It's an awful poem, doggerel in the rollicking meter of Robert W. Service's +The Cremation of Sam McGee, but it is in fact, a poem. It's the Ballad +of the Electronic Frontier! A poem about the Hacker Crackdown and the +sheer unlikelihood of CFP. It's full of in-jokes. The score or so cops +in the audience, who are sitting together in a nervous claque, +are absolutely cracking-up. Gail's poem is the funniest goddamn thing +they've ever heard. The hackers and civil-libs, who had this woman figured +for Ilsa She-Wolf of the SS, are staring with their jaws hanging loosely. +Never in the wildest reaches of their imagination had they figured +Gail Thackeray was capable of such a totally off-the-wall move. +You can see them punching their mental CONTROL-RESET buttons. +Jesus! This woman's a hacker weirdo! She's JUST LIKE US! +God, this changes everything! + +Al Bayse, computer technician for the FBI, had been the only cop +at the CPSR Roundtable, dragged there with his arm bent by +Dorothy Denning. He was guarded and tightlipped at CPSR Roundtable; +a "lion thrown to the Christians." + +At CFP, backed by a claque of cops, Bayse suddenly waxes eloquent +and even droll, describing the FBI's "NCIC 2000", a gigantic digital catalog +of criminal records, as if he has suddenly become some weird hybrid +of George Orwell and George Gobel. Tentatively, he makes an arcane +joke about statistical analysis. At least a third of the crowd laughs aloud. + +"They didn't laugh at that at my last speech," Bayse observes. +He had been addressing cops--STRAIGHT cops, not computer people. +It had been a worthy meeting, useful one supposes, but nothing like THIS. +There has never been ANYTHING like this. Without any prodding, +without any preparation, people in the audience simply begin to ask questions. +Longhairs, freaky people, mathematicians. Bayse is answering, politely, +frankly, fully, like a man walking on air. The ballroom's atmosphere +crackles with surreality. A female lawyer behind me breaks into a sweat +and a hot waft of surprisingly potent and musky perfume flows off +her pulse-points. + +People are giddy with laughter. People are interested, +fascinated, their eyes so wide and dark that they seem eroticized. +Unlikely daisy-chains form in the halls, around the bar, on the escalators: +cops with hackers, civil rights with FBI, Secret Service with phone phreaks. + +Gail Thackeray is at her crispest in a white wool sweater with a +tiny Secret Service logo. "I found Phiber Optik at the payphones, +and when he saw my sweater, he turned into a PILLAR OF SALT!" she chortles. + +Phiber discusses his case at much length with his arresting officer, +Don Delaney of the New York State Police. After an hour's chat, +the two of them look ready to begin singing "Auld Lang Syne." +Phiber finally finds the courage to get his worst complaint off his chest. +It isn't so much the arrest. It was the CHARGE. Pirating service +off 900 numbers. I'm a PROGRAMMER, Phiber insists. This lame charge +is going to hurt my reputation. It would have been cool to be busted +for something happening, like Section 1030 computer intrusion. +Maybe some kind of crime that's scarcely been invented yet. +Not lousy phone fraud. Phooey. + +Delaney seems regretful. He had a mountain of possible criminal charges +against Phiber Optik. The kid's gonna plead guilty anyway. He's a +first timer, they always plead. Coulda charged the kid with most anything, +and gotten the same result in the end. Delaney seems genuinely sorry +not to have gratified Phiber in this harmless fashion. Too late now. +Phiber's pled already. All water under the bridge. Whaddya gonna do? + +Delaney's got a good grasp on the hacker mentality. +He held a press conference after he busted a bunch of +Masters of Deception kids. Some journo had asked him: +"Would you describe these people as GENIUSES?" +Delaney's deadpan answer, perfect: "No, I would describe +these people as DEFENDANTS." Delaney busts a kid for +hacking codes with repeated random dialling. Tells the +press that NYNEX can track this stuff in no time flat nowadays, +and a kid has to be STUPID to do something so easy to catch. +Dead on again: hackers don't mind being thought of as Genghis Khan +by the straights, but if there's anything that really gets 'em +where they live, it's being called DUMB. + +Won't be as much fun for Phiber next time around. +As a second offender he's gonna see prison. +Hackers break the law. They're not geniuses, either. +They're gonna be defendants. And yet, Delaney muses over +a drink in the hotel bar, he has found it impossible to treat +them as common criminals. Delaney knows criminals. These kids, +by comparison, are clueless--there is just no crook vibe off of them, +they don't smell right, they're just not BAD. + +Delaney has seen a lot of action. He did Vietnam. +He's been shot at, he has shot people. He's a homicide +cop from New York. He has the appearance of a man who +has not only seen the shit hit the fan but has seen it splattered +across whole city blocks and left to ferment for years. +This guy has been around. + +He listens to Steve Jackson tell his story. The dreamy +game strategist has been dealt a bad hand. He has played +it for all he is worth. Under his nerdish SF-fan exterior +is a core of iron. Friends of his say Steve Jackson believes +in the rules, believes in fair play. He will never compromise +his principles, never give up. "Steve," Delaney says to +Steve Jackson, "they had some balls, whoever busted you. +You're all right!" Jackson, stunned, falls silent and +actually blushes with pleasure. + +Neidorf has grown up a lot in the past year. The kid is +a quick study, you gotta give him that. Dressed by his mom, +the fashion manager for a national clothing chain, +Missouri college techie-frat Craig Neidorf out-dappers +everyone at this gig but the toniest East Coast lawyers. +The iron jaws of prison clanged shut without him and now +law school beckons for Neidorf. He looks like a larval Congressman. + +Not a "hacker," our Mr. Neidorf. He's not interested +in computer science. Why should he be? He's not +interested in writing C code the rest of his life, +and besides, he's seen where the chips fall. +To the world of computer science he and Phrack +were just a curiosity. But to the world of law. . . . +The kid has learned where the bodies are buried. +He carries his notebook of press clippings wherever he goes. + +Phiber Optik makes fun of Neidorf for a Midwestern geek, +for believing that "Acid Phreak" does acid and listens to acid rock. +Hell no. Acid's never done ACID! Acid's into ACID HOUSE MUSIC. +Jesus. The very idea of doing LSD. Our PARENTS did LSD, ya clown. + +Thackeray suddenly turns upon Craig Neidorf the full lighthouse +glare of her attention and begins a determined half-hour attempt +to WIN THE BOY OVER. The Joan of Arc of Computer Crime is +GIVING CAREER ADVICE TO KNIGHT LIGHTNING! "Your experience +would be very valuable--a real asset," she tells him with +unmistakeable sixty-thousand-watt sincerity. Neidorf is fascinated. +He listens with unfeigned attention. He's nodding and saying yes ma'am. +Yes, Craig, you too can forget all about money and enter the glamorous +and horribly underpaid world of PROSECUTING COMPUTER CRIME! +You can put your former friends in prison--ooops. . . . + +You cannot go on dueling at modem's length indefinitely. +You cannot beat one another senseless with rolled-up press-clippings. +Sooner or later you have to come directly to grips. +And yet the very act of assembling here has changed +the entire situation drastically. John Quarterman, +author of The Matrix, explains the Internet at his symposium. +It is the largest news network in the world, it is growing +by leaps and bounds, and yet you cannot measure Internet because +you cannot stop it in place. It cannot stop, because there +is no one anywhere in the world with the authority to stop Internet. +It changes, yes, it grows, it embeds itself across the post-industrial, +postmodern world and it generates community wherever it +touches, and it is doing this all by itself. + +Phiber is different. A very fin de siecle kid, Phiber Optik. +Barlow says he looks like an Edwardian dandy. He does rather. +Shaven neck, the sides of his skull cropped hip-hop close, +unruly tangle of black hair on top that looks pomaded, +he stays up till four a.m. and misses all the sessions, +then hangs out in payphone booths with his acoustic coupler +gutsily CRACKING SYSTEMS RIGHT IN THE MIDST OF THE HEAVIEST +LAW ENFORCEMENT DUDES IN THE U.S., or at least PRETENDING to. . . . +Unlike "Frank Drake." Drake, who wrote Dorothy Denning out +of nowhere, and asked for an interview for his cheapo +cyberpunk fanzine, and then started grilling her on her ethics. +She was squirmin', too. . . . Drake, scarecrow-tall with his +floppy blond mohawk, rotting tennis shoes and black leather jacket +lettered ILLUMINATI in red, gives off an unmistakeable air +of the bohemian literatus. Drake is the kind of guy +who reads British industrial design magazines and appreciates +William Gibson because the quality of the prose is so tasty. +Drake could never touch a phone or a keyboard again, +and he'd still have the nose-ring and the blurry photocopied +fanzines and the sampled industrial music. He's a radical punk +with a desktop-publishing rig and an Internet address. +Standing next to Drake, the diminutive Phiber looks like he's +been physically coagulated out of phone-lines. Born to phreak. + +Dorothy Denning approaches Phiber suddenly. The two of them +are about the same height and body-build. Denning's blue eyes +flash behind the round window-frames of her glasses. +"Why did you say I was `quaint?'" she asks Phiber, quaintly. + +It's a perfect description but Phiber is nonplussed. . . +"Well, I uh, you know. . . ." + +"I also think you're quaint, Dorothy," I say, novelist to the rescue, +the journo gift of gab. . . . She is neat and dapper and yet there's +an arcane quality to her, something like a Pilgrim Maiden behind +leaded glass; if she were six inches high Dorothy Denning would look +great inside a china cabinet. . .The Cryptographeress. . . +The Cryptographrix. . .whatever. . . . Weirdly, Peter Denning looks +just like his wife, you could pick this gentleman out of a thousand guys +as the soulmate of Dorothy Denning. Wearing tailored slacks, +a spotless fuzzy varsity sweater, and a neatly knotted academician's tie. . . . +This fineboned, exquisitely polite, utterly civilized and hyperintelligent +couple seem to have emerged from some cleaner and finer parallel universe, +where humanity exists to do the Brain Teasers column in Scientific American. +Why does this Nice Lady hang out with these unsavory characters? + +Because the time has come for it, that's why. +Because she's the best there is at what she does. + +Donn Parker is here, the Great Bald Eagle of Computer Crime. . . . +With his bald dome, great height, and enormous Lincoln-like hands, +the great visionary pioneer of the field plows through the lesser mortals +like an icebreaker. . . . His eyes are fixed on the future with the +rigidity of a bronze statue. . . . Eventually, he tells his audience, +all business crime will be computer crime, because businesses will do +everything through computers. "Computer crime" as a category will vanish. + +In the meantime, passing fads will flourish and fail and evaporate. . . . +Parker's commanding, resonant voice is sphinxlike, everything is viewed +from some eldritch valley of deep historical abstraction. . . . +Yes, they've come and they've gone, these passing flaps in the world +of digital computation. . . . The radio-frequency emanation scandal. . . +KGB and MI5 and CIA do it every day, it's easy, but nobody else ever has. . . . +The salami-slice fraud, mostly mythical. . . . "Crimoids," he calls them. . . . +Computer viruses are the current crimoid champ, a lot less dangerous than +most people let on, but the novelty is fading and there's a crimoid vacuum at +the moment, the press is visibly hungering for something more outrageous. . . . +The Great Man shares with us a few speculations on the coming crimoids. . . . +Desktop Forgery! Wow. . . . Computers stolen just for the sake of the +information within them--data-napping! Happened in Britain a while ago, +could be the coming thing. . . . Phantom nodes in the Internet! + +Parker handles his overhead projector sheets with an ecclesiastical air. . . . +He wears a grey double-breasted suit, a light blue shirt, and a +very quiet tie of understated maroon and blue paisley. . . . +Aphorisms emerge from him with slow, leaden emphasis. . . . +There is no such thing as an adequately secure computer +when one faces a sufficiently powerful adversary. . . . +Deterrence is the most socially useful aspect of security. . . . +People are the primary weakness in all information systems. . . . +The entire baseline of computer security must be shifted upward. . . . +Don't ever violate your security by publicly describing +your security measures. . . . + +People in the audience are beginning to squirm, and yet +there is something about the elemental purity of this guy's +philosophy that compels uneasy respect. . . . Parker sounds +like the only sane guy left in the lifeboat, sometimes. +The guy who can prove rigorously, from deep moral principles, +that Harvey there, the one with the broken leg and the checkered past, +is the one who has to be, err. . .that is, Mr. Harvey is best placed +to make the necessary sacrifice for the security and indeed +the very survival of the rest of this lifeboat's crew. . . . +Computer security, Parker informs us mournfully, is a +nasty topic, and we wish we didn't have to have it. . . . +The security expert, armed with method and logic, must think--imagine-- +everything that the adversary might do before the adversary might +actually do it. It is as if the criminal's dark brain were an +extensive subprogram within the shining cranium of Donn Parker. +He is a Holmes whose Moriarty does not quite yet exist +and so must be perfectly simulated. + +CFP is a stellar gathering, with the giddiness of a wedding. +It is a happy time, a happy ending, they know their world +is changing forever tonight, and they're proud to have been there +to see it happen, to talk, to think, to help. + +And yet as night falls, a certain elegiac quality manifests itself, +as the crowd gathers beneath the chandeliers with their wineglasses +and dessert plates. Something is ending here, gone forever, +and it takes a while to pinpoint it. + +It is the End of the Amateurs. + + + + + + + + + +End of the Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling + +*** END OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** + +***** This file should be named 101.txt or 101.zip ***** +This and all associated files of various formats will be found in: + http://www.gutenberg.org/1/0/101/ + + + +Updated editions will replace the previous one--the old editions will be +renamed. + +Creating the works from public domain print editions means that no one +owns a United States copyright in these works, so the Foundation (and +you!) can copy and distribute it in the United States without permission +and without paying copyright royalties. Special rules, set forth in the +General Terms of Use part of this license, apply to copying and +distributing Project Gutenberg-tm electronic works to protect the +PROJECT GUTENBERG-tm concept and trademark. Project Gutenberg is a +registered trademark, and may not be used if you charge for the eBooks, +unless you receive specific permission. If you do not charge anything +for copies of this eBook, complying with the rules is very easy. You may +use this eBook for nearly any purpose such as creation of derivative +works, reports, performances and research. They may be modified and +printed and given away--you may do practically ANYTHING with public +domain eBooks. Redistribution is subject to the trademark license, +especially commercial redistribution. + + + +*** START: FULL LICENSE *** + +THE FULL PROJECT GUTENBERG LICENSE +PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK + +To protect the Project Gutenberg-tm mission of promoting the free +distribution of electronic works, by using or distributing this work +(or any other work associated in any way with the phrase "Project +Gutenberg"), you agree to comply with all the terms of the Full Project +Gutenberg-tm License (available with this file or online at +http://www.gutenberg.org/license). + + +Section 1. General Terms of Use and Redistributing Project Gutenberg-tm +electronic works + +1.A. By reading or using any part of this Project Gutenberg-tm +electronic work, you indicate that you have read, understand, agree to +and accept all the terms of this license and intellectual property +(trademark/copyright) agreement. If you do not agree to abide by all +the terms of this agreement, you must cease using and return or destroy +all copies of Project Gutenberg-tm electronic works in your possession. +If you paid a fee for obtaining a copy of or access to a Project +Gutenberg-tm electronic work and you do not agree to be bound by the +terms of this agreement, you may obtain a refund from the person or +entity to whom you paid the fee as set forth in paragraph 1.E.8. + +1.B. "Project Gutenberg" is a registered trademark. It may only be +used on or associated in any way with an electronic work by people who +agree to be bound by the terms of this agreement. There are a few +things that you can do with most Project Gutenberg-tm electronic works +even without complying with the full terms of this agreement. See +paragraph 1.C below. There are a lot of things you can do with Project +Gutenberg-tm electronic works if you follow the terms of this agreement +and help preserve free future access to Project Gutenberg-tm electronic +works. See paragraph 1.E below. + +1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation" +or PGLAF), owns a compilation copyright in the collection of Project +Gutenberg-tm electronic works. Nearly all the individual works in the +collection are in the public domain in the United States. If an +individual work is in the public domain in the United States and you are +located in the United States, we do not claim a right to prevent you from +copying, distributing, performing, displaying or creating derivative +works based on the work as long as all references to Project Gutenberg +are removed. Of course, we hope that you will support the Project +Gutenberg-tm mission of promoting free access to electronic works by +freely sharing Project Gutenberg-tm works in compliance with the terms of +this agreement for keeping the Project Gutenberg-tm name associated with +the work. You can easily comply with the terms of this agreement by +keeping this work in the same format with its attached full Project +Gutenberg-tm License when you share it without charge with others. +This particular work is one of the few copyrighted individual works +included with the permission of the copyright holder. Information on +the copyright owner for this particular work and the terms of use +imposed by the copyright holder on this work are set forth at the +beginning of this work. + +1.D. The copyright laws of the place where you are located also govern +what you can do with this work. Copyright laws in most countries are in +a constant state of change. If you are outside the United States, check +the laws of your country in addition to the terms of this agreement +before downloading, copying, displaying, performing, distributing or +creating derivative works based on this work or any other Project +Gutenberg-tm work. The Foundation makes no representations concerning +the copyright status of any work in any country outside the United +States. + +1.E. Unless you have removed all references to Project Gutenberg: + +1.E.1. The following sentence, with active links to, or other immediate +access to, the full Project Gutenberg-tm License must appear prominently +whenever any copy of a Project Gutenberg-tm work (any work on which the +phrase "Project Gutenberg" appears, or with which the phrase "Project +Gutenberg" is associated) is accessed, displayed, performed, viewed, +copied or distributed: + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + +1.E.2. If an individual Project Gutenberg-tm electronic work is derived +from the public domain (does not contain a notice indicating that it is +posted with permission of the copyright holder), the work can be copied +and distributed to anyone in the United States without paying any fees +or charges. If you are redistributing or providing access to a work +with the phrase "Project Gutenberg" associated with or appearing on the +work, you must comply either with the requirements of paragraphs 1.E.1 +through 1.E.7 or obtain permission for the use of the work and the +Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or +1.E.9. + +1.E.3. If an individual Project Gutenberg-tm electronic work is posted +with the permission of the copyright holder, your use and distribution +must comply with both paragraphs 1.E.1 through 1.E.7 and any additional +terms imposed by the copyright holder. Additional terms will be linked +to the Project Gutenberg-tm License for all works posted with the +permission of the copyright holder found at the beginning of this work. + +1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm +License terms from this work, or any files containing a part of this +work or any other work associated with Project Gutenberg-tm. + +1.E.5. Do not copy, display, perform, distribute or redistribute this +electronic work, or any part of this electronic work, without +prominently displaying the sentence set forth in paragraph 1.E.1 with +active links or immediate access to the full terms of the Project +Gutenberg-tm License. + +1.E.6. You may convert to and distribute this work in any binary, +compressed, marked up, nonproprietary or proprietary form, including any +word processing or hypertext form. However, if you provide access to or +distribute copies of a Project Gutenberg-tm work in a format other than +"Plain Vanilla ASCII" or other format used in the official version +posted on the official Project Gutenberg-tm web site (www.gutenberg.org), +you must, at no additional cost, fee or expense to the user, provide a +copy, a means of exporting a copy, or a means of obtaining a copy upon +request, of the work in its original "Plain Vanilla ASCII" or other +form. Any alternate format must include the full Project Gutenberg-tm +License as specified in paragraph 1.E.1. + +1.E.7. Do not charge a fee for access to, viewing, displaying, +performing, copying or distributing any Project Gutenberg-tm works +unless you comply with paragraph 1.E.8 or 1.E.9. + +1.E.8. You may charge a reasonable fee for copies of or providing +access to or distributing Project Gutenberg-tm electronic works provided +that + +- You pay a royalty fee of 20% of the gross profits you derive from + the use of Project Gutenberg-tm works calculated using the method + you already use to calculate your applicable taxes. The fee is + owed to the owner of the Project Gutenberg-tm trademark, but he + has agreed to donate royalties under this paragraph to the + Project Gutenberg Literary Archive Foundation. Royalty payments + must be paid within 60 days following each date on which you + prepare (or are legally required to prepare) your periodic tax + returns. Royalty payments should be clearly marked as such and + sent to the Project Gutenberg Literary Archive Foundation at the + address specified in Section 4, "Information about donations to + the Project Gutenberg Literary Archive Foundation." + +- You provide a full refund of any money paid by a user who notifies + you in writing (or by e-mail) within 30 days of receipt that s/he + does not agree to the terms of the full Project Gutenberg-tm + License. You must require such a user to return or + destroy all copies of the works possessed in a physical medium + and discontinue all use of and all access to other copies of + Project Gutenberg-tm works. + +- You provide, in accordance with paragraph 1.F.3, a full refund of any + money paid for a work or a replacement copy, if a defect in the + electronic work is discovered and reported to you within 90 days + of receipt of the work. + +- You comply with all other terms of this agreement for free + distribution of Project Gutenberg-tm works. + +1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm +electronic work or group of works on different terms than are set +forth in this agreement, you must obtain permission in writing from +both the Project Gutenberg Literary Archive Foundation and Michael +Hart, the owner of the Project Gutenberg-tm trademark. Contact the +Foundation as set forth in Section 3 below. + +1.F. + +1.F.1. Project Gutenberg volunteers and employees expend considerable +effort to identify, do copyright research on, transcribe and proofread +public domain works in creating the Project Gutenberg-tm +collection. Despite these efforts, Project Gutenberg-tm electronic +works, and the medium on which they may be stored, may contain +"Defects," such as, but not limited to, incomplete, inaccurate or +corrupt data, transcription errors, a copyright or other intellectual +property infringement, a defective or damaged disk or other medium, a +computer virus, or computer codes that damage or cannot be read by +your equipment. + +1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right +of Replacement or Refund" described in paragraph 1.F.3, the Project +Gutenberg Literary Archive Foundation, the owner of the Project +Gutenberg-tm trademark, and any other party distributing a Project +Gutenberg-tm electronic work under this agreement, disclaim all +liability to you for damages, costs and expenses, including legal +fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT +LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE +PROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE +TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE +LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR +INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH +DAMAGE. + +1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a +defect in this electronic work within 90 days of receiving it, you can +receive a refund of the money (if any) you paid for it by sending a +written explanation to the person you received the work from. If you +received the work on a physical medium, you must return the medium with +your written explanation. The person or entity that provided you with +the defective work may elect to provide a replacement copy in lieu of a +refund. If you received the work electronically, the person or entity +providing it to you may choose to give you a second opportunity to +receive the work electronically in lieu of a refund. If the second copy +is also defective, you may demand a refund in writing without further +opportunities to fix the problem. + +1.F.4. Except for the limited right of replacement or refund set forth +in paragraph 1.F.3, this work is provided to you 'AS-IS,' WITH NO OTHER +WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE. + +1.F.5. Some states do not allow disclaimers of certain implied +warranties or the exclusion or limitation of certain types of damages. +If any disclaimer or limitation set forth in this agreement violates the +law of the state applicable to this agreement, the agreement shall be +interpreted to make the maximum disclaimer or limitation permitted by +the applicable state law. The invalidity or unenforceability of any +provision of this agreement shall not void the remaining provisions. + +1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the +trademark owner, any agent or employee of the Foundation, anyone +providing copies of Project Gutenberg-tm electronic works in accordance +with this agreement, and any volunteers associated with the production, +promotion and distribution of Project Gutenberg-tm electronic works, +harmless from all liability, costs and expenses, including legal fees, +that arise directly or indirectly from any of the following which you do +or cause to occur: (a) distribution of this or any Project Gutenberg-tm +work, (b) alteration, modification, or additions or deletions to any +Project Gutenberg-tm work, and (c) any Defect you cause. + + +Section 2. Information about the Mission of Project Gutenberg-tm + +Project Gutenberg-tm is synonymous with the free distribution of +electronic works in formats readable by the widest variety of computers +including obsolete, old, middle-aged and new computers. It exists +because of the efforts of hundreds of volunteers and donations from +people in all walks of life. + +Volunteers and financial support to provide volunteers with the +assistance they need are critical to reaching Project Gutenberg-tm's +goals and ensuring that the Project Gutenberg-tm collection will +remain freely available for generations to come. In 2001, the Project +Gutenberg Literary Archive Foundation was created to provide a secure +and permanent future for Project Gutenberg-tm and future generations. +To learn more about the Project Gutenberg Literary Archive Foundation +and how your efforts and donations can help, see Sections 3 and 4 +and the Foundation web page at http://www.pglaf.org. + + +Section 3. Information about the Project Gutenberg Literary Archive +Foundation + +The Project Gutenberg Literary Archive Foundation is a non profit +501(c)(3) educational corporation organized under the laws of the +state of Mississippi and granted tax exempt status by the Internal +Revenue Service. The Foundation's EIN or federal tax identification +number is 64-6221541. Its 501(c)(3) letter is posted at +http://pglaf.org/fundraising. Contributions to the Project Gutenberg +Literary Archive Foundation are tax deductible to the full extent +permitted by U.S. federal laws and your state's laws. + +The Foundation's principal office is located at 4557 Melan Dr. S. +Fairbanks, AK, 99712., but its volunteers and employees are scattered +throughout numerous locations. Its business office is located at +809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email +business@pglaf.org. Email contact links and up to date contact +information can be found at the Foundation's web site and official +page at http://pglaf.org + +For additional contact information: + Dr. Gregory B. Newby + Chief Executive and Director + gbnewby@pglaf.org + +Section 4. Information about Donations to the Project Gutenberg +Literary Archive Foundation + +Project Gutenberg-tm depends upon and cannot survive without wide +spread public support and donations to carry out its mission of +increasing the number of public domain and licensed works that can be +freely distributed in machine readable form accessible by the widest +array of equipment including outdated equipment. Many small donations +($1 to $5,000) are particularly important to maintaining tax exempt +status with the IRS. + +The Foundation is committed to complying with the laws regulating +charities and charitable donations in all 50 states of the United +States. Compliance requirements are not uniform and it takes a +considerable effort, much paperwork and many fees to meet and keep up +with these requirements. We do not solicit donations in locations +where we have not received written confirmation of compliance. To +SEND DONATIONS or determine the status of compliance for any +particular state visit http://pglaf.org + +While we cannot and do not solicit contributions from states where we +have not met the solicitation requirements, we know of no prohibition +against accepting unsolicited donations from donors in such states who +approach us with offers to donate. + +International donations are gratefully accepted, but we cannot make +any statements concerning tax treatment of donations received from +outside the United States. U.S. laws alone swamp our small staff. + +Please check the Project Gutenberg Web pages for current donation +methods and addresses. Donations are accepted in a number of other +ways including checks, online payments and credit card donations. +To donate, please visit: http://pglaf.org/donate + + +Section 5. General Information About Project Gutenberg-tm electronic +works. + +Professor Michael S. Hart is the originator of the Project Gutenberg-tm +concept of a library of electronic works that could be freely shared +with anyone. For thirty years, he produced and distributed Project +Gutenberg-tm eBooks with only a loose network of volunteer support. + +Project Gutenberg-tm eBooks are often created from several printed +editions, all of which are confirmed as Public Domain in the U.S. +unless a copyright notice is included. Thus, we do not necessarily +keep eBooks in compliance with any particular paper edition. + +Each eBook is in a subdirectory of the same number as the eBook's +eBook number, often in several formats including plain vanilla ASCII, +compressed (zipped), HTML and others. + +Corrected EDITIONS of our eBooks replace the old file and take over +the old filename and etext number. The replaced older file is renamed. +VERSIONS based on separate sources are treated as new eBooks receiving +new filenames and etext numbers. + +Most people start at our Web site which has the main PG search facility: + +http://www.gutenberg.org + +This Web site includes information about Project Gutenberg-tm, +including how to make donations to the Project Gutenberg Literary +Archive Foundation, how to help produce our new eBooks, and how to +subscribe to our email newsletter to hear about new eBooks. + +EBooks posted prior to November 2003, with eBook numbers BELOW #10000, +are filed in directories based on their release date. If you want to +download any of these eBooks directly, rather than using the regular +search system you may utilize the following addresses and just +download by the etext year. + +http://www.ibiblio.org/gutenberg/etext06 + + (Or /etext 05, 04, 03, 02, 01, 00, 99, + 98, 97, 96, 95, 94, 93, 92, 92, 91 or 90) + +EBooks posted since November 2003, with etext numbers OVER #10000, are +filed in a different way. The year of a release date is no longer part +of the directory path. The path is based on the etext number (which is +identical to the filename). The path to the file is made up of single +digits corresponding to all but the last digit in the filename. For +example an eBook of filename 10234 would be found at: + +http://www.gutenberg.org/1/0/2/3/10234 + +or filename 24689 would be found at: +http://www.gutenberg.org/2/4/6/8/24689 + +An alternative method of locating eBooks: +http://www.gutenberg.org/GUTINDEX.ALL + +*** END: FULL LICENSE *** diff --git a/common/src/leap/soledad/common/tests/test_async.py b/common/src/leap/soledad/common/tests/test_async.py new file mode 100644 index 00000000..03b8c553 --- /dev/null +++ b/common/src/leap/soledad/common/tests/test_async.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# test_async.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os +import hashlib + +from twisted.internet import defer + +from leap.soledad.common.tests.util import BaseSoledadTest +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +class ASyncSQLCipherRetryTestCase(BaseSoledadTest): + """ + Test asynchronous SQLCipher operation. + """ + + NUM_DOCS = 5000 + + def _get_dbpool(self): + tmpdb = os.path.join(self.tempdir, "test.soledad") + opts = SQLCipherOptions(tmpdb, "secret", create=True) + return adbapi.getConnectionPool(opts) + + def _get_sample(self): + if not getattr(self, "_sample", None): + dirname = os.path.dirname(os.path.realpath(__file__)) + sample_file = os.path.join(dirname, "hacker_crackdown.txt") + with open(sample_file) as f: + self._sample = f.readlines() + return self._sample + + def test_concurrent_puts_fail_with_few_retries_and_small_timeout(self): + """ + Test if concurrent updates to the database with small timeout and + small number of retries fail with "database is locked" error. + + Many concurrent write attempts to the same sqlcipher database may fail + when the timeout is small and there are no retries. This test will + pass if any of the attempts to write the database fail. + + This test is much dependent on the environment and its result intends + to contrast with the test for the workaround for the "database is + locked" problem, which is addressed by the "test_concurrent_puts" test + below. + + If this test ever fails, it means that either (1) the platform where + you are running is it very powerful and you should try with an even + lower timeout value, or (2) the bug has been solved by a better + implementation of the underlying database pool, and thus this test + should be removed from the test suite. + """ + + old_timeout = adbapi.SQLCIPHER_CONNECTION_TIMEOUT + old_max_retries = adbapi.SQLCIPHER_MAX_RETRIES + + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1 + adbapi.SQLCIPHER_MAX_RETRIES = 1 + + dbpool = self._get_dbpool() + + def _create_doc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def _errback(e): + if e.value[0].getErrorMessage() == "database is locked": + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = old_timeout + adbapi.SQLCIPHER_MAX_RETRIES = old_max_retries + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) + d.addErrback(_errback) + return d + + def test_concurrent_puts(self): + """ + Test that many concurrent puts succeed. + + Currently, there's a known problem with the concurrent database pool + which is that many concurrent attempts to write to the database may + fail when the lock timeout is small and when there are no (or few) + retries. We currently workaround this problem by increasing the + timeout and the number of retries. + + Should this test ever fail, it probably means that the timeout and/or + number of retries should be increased for the platform you're running + the test. If the underlying database pool is ever fixed, then the test + above will fail and we should remove this comment from here. + """ + + dbpool = self._get_dbpool() + + def _create_doc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + + def _count_docs(results): + _, docs = results + if self.NUM_DOCS == len(docs): + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) + d.addCallback(_count_docs) + return d -- cgit v1.2.3 From 41b34cc0d8bd6c2ae22547bc02845e68cab12c3b Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 20 Feb 2015 16:01:57 -0400 Subject: cutoff for encoding detection --- client/changes/bug_cutoff-chardet-guessing | 1 + client/src/leap/soledad/client/api.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 client/changes/bug_cutoff-chardet-guessing diff --git a/client/changes/bug_cutoff-chardet-guessing b/client/changes/bug_cutoff-chardet-guessing new file mode 100644 index 00000000..9535a413 --- /dev/null +++ b/client/changes/bug_cutoff-chardet-guessing @@ -0,0 +1 @@ +- Fallback to utf-8 if confidence on chardet guessing is too low. diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 88bb4969..b8409cbe 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -416,6 +416,10 @@ class Soledad(object): :return: A deferred whose callback will be invoked with a document. :rtype: twisted.internet.defer.Deferred """ + # TODO we probably should pass an optional "encoding" parameter to + # create_doc (and probably to put_doc too). There are cases (mail + # payloads for example) in which we already have the encoding in the + # headers, so we don't need to guess it. return self._defer( "create_doc", _convert_to_unicode(content), doc_id=doc_id) @@ -803,12 +807,17 @@ def _convert_to_unicode(content): :rtype: object """ + # Chardet doesn't guess very well with some smallish payloads. + # This parameter might need some empirical tweaking. + CUTOFF_CONFIDENCE = 0.90 + if isinstance(content, unicode): return content elif isinstance(content, str): + encoding = "utf-8" result = chardet.detect(content) - default = "utf-8" - encoding = result["encoding"] or default + if result["confidence"] > CUTOFF_CONFIDENCE: + encoding = result["encoding"] try: content = content.decode(encoding) except UnicodeError as e: -- cgit v1.2.3 From 61a56f2ee301212d96c2d95a21d524bc06b3a677 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 9 Mar 2015 15:22:17 -0300 Subject: Fix soledad initscript uid and gid. --- server/changes/bug_fix-initscript-uid-and-gid | 1 + server/pkg/soledad-server | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 server/changes/bug_fix-initscript-uid-and-gid diff --git a/server/changes/bug_fix-initscript-uid-and-gid b/server/changes/bug_fix-initscript-uid-and-gid new file mode 100644 index 00000000..d4767984 --- /dev/null +++ b/server/changes/bug_fix-initscript-uid-and-gid @@ -0,0 +1 @@ + o Fix server daemon uid and gid by passing them to twistd on the initscript. diff --git a/server/pkg/soledad-server b/server/pkg/soledad-server index ccb3e9b0..811ad55b 100644 --- a/server/pkg/soledad-server +++ b/server/pkg/soledad-server @@ -34,8 +34,8 @@ case "${1}" in start) echo -n "Starting soledad: twistd" start-stop-daemon --start --quiet \ - --user=${USER} --group=${GROUP} \ --exec ${TWISTD_PATH} -- \ + --uid=${USER} --gid=${GROUP} \ --pidfile=${PIDFILE} \ --logfile=${LOGFILE} \ web \ -- cgit v1.2.3 From cf3c5018820f982ae64c2e062391b0a3b6e52f21 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 11 Mar 2015 14:33:10 -0300 Subject: [feat] use monthly tokens database Any solead release that includes this commit will be incompatible with LEAP Platform < 0.6.1 because only from that version on the platform implements the ephemeral monthly tokens databases. Closes: #6785. --- README.rst | 8 ++++++-- server/changes/feature_6785_use-monthly-token-db | 1 + server/src/leap/soledad/server/auth.py | 22 +++++++++++++--------- 3 files changed, 20 insertions(+), 11 deletions(-) create mode 100644 server/changes/feature_6785_use-monthly-token-db diff --git a/README.rst b/README.rst index 5d6593ca..887b3df1 100644 --- a/README.rst +++ b/README.rst @@ -30,8 +30,12 @@ repository: Compatibility ------------- -* Server 0.7.x is incompatible with client < 0.7.0 because of modifications on - encrypted document MAC calculation. +* Soledad Server >= 0.7.0 is incompatible with client < 0.7.0 because of + modifications on encrypted document MAC calculation. + +* Soledad Server >= 0.7.0 is incompatible with LEAP Platform < 0.6.1 because + that platform version implements ephemeral tokens databases and Soledad + Server needs to act accordingly. Tests diff --git a/server/changes/feature_6785_use-monthly-token-db b/server/changes/feature_6785_use-monthly-token-db new file mode 100644 index 00000000..f7987cad --- /dev/null +++ b/server/changes/feature_6785_use-monthly-token-db @@ -0,0 +1 @@ + o Use monthly token databases. Closes #6785. diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 57f600a1..7af4e54b 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -21,10 +21,10 @@ Authentication facilities for Soledad Server. """ +import time import httplib import simplejson as json - from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors from abc import ABCMeta, abstractmethod from routes.mapper import Mapper @@ -32,12 +32,8 @@ from couchdb.client import Server from twisted.python import log from hashlib import sha512 - -from leap.soledad.common import ( - SHARED_DB_NAME, - SHARED_DB_LOCK_DOC_ID_PREFIX, - USER_DB_PREFIX, -) +from leap.soledad.common import SHARED_DB_NAME +from leap.soledad.common import USER_DB_PREFIX from leap.soledad.common.errors import InvalidAuthTokenError @@ -354,7 +350,8 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): Token based authentication. """ - TOKENS_DB = "tokens" + TOKENS_DB_PREFIX = "tokens_" + TOKENS_DB_EXPIRE = 30 * 24 * 3600 # 30 days in seconds TOKENS_TYPE_KEY = "type" TOKENS_TYPE_DEF = "Token" TOKENS_USER_ID_KEY = "user_id" @@ -414,7 +411,14 @@ class SoledadTokenAuthMiddleware(SoledadAuthMiddleware): invalid. """ server = Server(url=self._app.state.couch_url) - dbname = self.TOKENS_DB + # the tokens db rotates every 30 days, and the current db name is + # "tokens_NNN", where NNN is the number of seconds since epoch divided + # by the rotate period in seconds. When rotating, old and new tokens + # db coexist during a certain window of time and valid tokens are + # replicated from the old db to the new one. See: + # https://leap.se/code/issues/6785 + dbname = self.TOKENS_DB_PREFIX + \ + str(int(time.time() / self.TOKENS_DB_EXPIRE)) db = server[dbname] # lookup key is a hash of the token to prevent timing attacks. token = db.get(sha512(token).hexdigest()) -- cgit v1.2.3 From 4b78cf9da0874501fa123a02b53d7650e8dfcdf1 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 19 Mar 2015 09:54:38 -0300 Subject: [fix] add/fix dependency on twisted Add dependency on twisted for Soledad Client. Also remove minimum twisted version for Soledad Server because debian stable currently distributes 12.0.0 and pypi currently distributes 15.0.0. Closes: #6797 --- client/changes/bug_6797_add-dependency-on-twisted | 1 + client/pkg/requirements.pip | 1 + server/changes/bug_6797_add-dependency-on-twisted | 1 + server/pkg/requirements.pip | 4 +--- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 client/changes/bug_6797_add-dependency-on-twisted create mode 100644 server/changes/bug_6797_add-dependency-on-twisted diff --git a/client/changes/bug_6797_add-dependency-on-twisted b/client/changes/bug_6797_add-dependency-on-twisted new file mode 100644 index 00000000..962222b0 --- /dev/null +++ b/client/changes/bug_6797_add-dependency-on-twisted @@ -0,0 +1 @@ + o Add dependency on Twisted. Closes #6797. diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index 61258f01..33770adc 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -5,6 +5,7 @@ scrypt pycryptopp cchardet zope.proxy +twisted # # leap deps diff --git a/server/changes/bug_6797_add-dependency-on-twisted b/server/changes/bug_6797_add-dependency-on-twisted new file mode 100644 index 00000000..962222b0 --- /dev/null +++ b/server/changes/bug_6797_add-dependency-on-twisted @@ -0,0 +1 @@ + o Add dependency on Twisted. Closes #6797. diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index 28717664..89ec52e7 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -4,9 +4,7 @@ simplejson u1db routes PyOpenSSL<0.14 - -# TODO: maybe we just want twisted-web? -twisted>=12.0.0 +twisted # leap deps -- bump me! leap.soledad.common>=0.6.0 -- cgit v1.2.3 From 22adcae07584773a100bf304162113a9326a3866 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 19 Mar 2015 10:32:34 -0300 Subject: [fix] exclude all tests from package Previous to this modification, leap.soledad.common.tests.u1db_tests was being installed and its files were being included in the debian package. By excluding *.tests and *.tests.* from find_packages() in setup.py, we make sure that no test file will be installed not included in the final debian package. --- common/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/setup.py b/common/setup.py index b0ab8352..f4d8bc65 100644 --- a/common/setup.py +++ b/common/setup.py @@ -270,7 +270,7 @@ setup( ), classifiers=trove_classifiers, namespace_packages=["leap", "leap.soledad"], - packages=find_packages('src', exclude=['leap.soledad.common.tests']), + packages=find_packages('src', exclude=['*.tests', '*.tests.*']), package_dir={'': 'src'}, test_suite='leap.soledad.common.tests', install_requires=utils.parse_requirements(), -- cgit v1.2.3 From 74dec41c1d99ae8d4a4a79a7cb0d5c3c9f40cbae Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 19 Mar 2015 10:57:54 -0300 Subject: [fix] add explicit dependency on leap.common In the past, we wanted dependency on leap.common to be optional, but now because of the explicit use of the config path prefix and signaling, we want to enforce dependency on leap.common. --- client/pkg/requirements.pip | 15 +++---- client/src/leap/soledad/client/events.py | 67 ++++++++++++++---------------- common/pkg/requirements.pip | 9 ++-- common/src/leap/soledad/common/__init__.py | 46 ++++---------------- server/pkg/requirements.pip | 11 ++--- 5 files changed, 55 insertions(+), 93 deletions(-) diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index 33770adc..e00ab961 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -7,16 +7,11 @@ cchardet zope.proxy twisted -# -# leap deps -# - +# leap deps -- bump me! +leap.common leap.soledad.common>=0.6.0 -# -# XXX things to fix yet: -# - -# this is not strictly needed by us, but we need it -# until u1db adds it to its release as a dep. +# XXX -- fix me! +# oauth is not strictly needed by us, but we need it until u1db adds it to its +# release as a dep. oauth diff --git a/client/src/leap/soledad/client/events.py b/client/src/leap/soledad/client/events.py index c4c09ac5..88e28674 100644 --- a/client/src/leap/soledad/client/events.py +++ b/client/src/leap/soledad/client/events.py @@ -21,38 +21,35 @@ Signaling functions. """ -SOLEDAD_CREATING_KEYS = 'Creating keys...' -SOLEDAD_DONE_CREATING_KEYS = 'Done creating keys.' -SOLEDAD_DOWNLOADING_KEYS = 'Downloading keys...' -SOLEDAD_DONE_DOWNLOADING_KEYS = 'Done downloading keys.' -SOLEDAD_UPLOADING_KEYS = 'Uploading keys...' -SOLEDAD_DONE_UPLOADING_KEYS = 'Done uploading keys.' -SOLEDAD_NEW_DATA_TO_SYNC = 'New data available.' -SOLEDAD_DONE_DATA_SYNC = 'Done data sync.' -SOLEDAD_SYNC_SEND_STATUS = 'Sync: sent one document.' -SOLEDAD_SYNC_RECEIVE_STATUS = 'Sync: received one document.' - -# we want to use leap.common.events to emits signals, if it is available. -try: - from leap.common import events - from leap.common.events import signal - SOLEDAD_CREATING_KEYS = events.proto.SOLEDAD_CREATING_KEYS - SOLEDAD_DONE_CREATING_KEYS = events.proto.SOLEDAD_DONE_CREATING_KEYS - SOLEDAD_DOWNLOADING_KEYS = events.proto.SOLEDAD_DOWNLOADING_KEYS - SOLEDAD_DONE_DOWNLOADING_KEYS = \ - events.proto.SOLEDAD_DONE_DOWNLOADING_KEYS - SOLEDAD_UPLOADING_KEYS = events.proto.SOLEDAD_UPLOADING_KEYS - SOLEDAD_DONE_UPLOADING_KEYS = \ - events.proto.SOLEDAD_DONE_UPLOADING_KEYS - SOLEDAD_NEW_DATA_TO_SYNC = events.proto.SOLEDAD_NEW_DATA_TO_SYNC - SOLEDAD_DONE_DATA_SYNC = events.proto.SOLEDAD_DONE_DATA_SYNC - SOLEDAD_SYNC_SEND_STATUS = events.proto.SOLEDAD_SYNC_SEND_STATUS - SOLEDAD_SYNC_RECEIVE_STATUS = events.proto.SOLEDAD_SYNC_RECEIVE_STATUS - -except ImportError: - # we define a fake signaling function and fake signal constants that will - # allow for logging signaling attempts in case leap.common.events is not - # available. - - def signal(signal, content=""): - logger.info("Would signal: %s - %s." % (str(signal), content)) +from leap.common import events +from leap.common.events import signal + + +SOLEDAD_CREATING_KEYS = events.proto.SOLEDAD_CREATING_KEYS +SOLEDAD_DONE_CREATING_KEYS = events.proto.SOLEDAD_DONE_CREATING_KEYS +SOLEDAD_DOWNLOADING_KEYS = events.proto.SOLEDAD_DOWNLOADING_KEYS +SOLEDAD_DONE_DOWNLOADING_KEYS = \ + events.proto.SOLEDAD_DONE_DOWNLOADING_KEYS +SOLEDAD_UPLOADING_KEYS = events.proto.SOLEDAD_UPLOADING_KEYS +SOLEDAD_DONE_UPLOADING_KEYS = \ + events.proto.SOLEDAD_DONE_UPLOADING_KEYS +SOLEDAD_NEW_DATA_TO_SYNC = events.proto.SOLEDAD_NEW_DATA_TO_SYNC +SOLEDAD_DONE_DATA_SYNC = events.proto.SOLEDAD_DONE_DATA_SYNC +SOLEDAD_SYNC_SEND_STATUS = events.proto.SOLEDAD_SYNC_SEND_STATUS +SOLEDAD_SYNC_RECEIVE_STATUS = events.proto.SOLEDAD_SYNC_RECEIVE_STATUS + + +__all__ = [ + "events", + "signal", + "SOLEDAD_CREATING_KEYS", + "SOLEDAD_DONE_CREATING_KEYS", + "SOLEDAD_DOWNLOADING_KEYS", + "SOLEDAD_DONE_DOWNLOADING_KEYS", + "SOLEDAD_UPLOADING_KEYS", + "SOLEDAD_DONE_UPLOADING_KEYS", + "SOLEDAD_NEW_DATA_TO_SYNC", + "SOLEDAD_DONE_DATA_SYNC", + "SOLEDAD_SYNC_SEND_STATUS", + "SOLEDAD_SYNC_RECEIVE_STATUS", +] diff --git a/common/pkg/requirements.pip b/common/pkg/requirements.pip index 5787114e..ea2f3fa2 100644 --- a/common/pkg/requirements.pip +++ b/common/pkg/requirements.pip @@ -1,7 +1,10 @@ simplejson u1db -#this is not strictly needed by us, but we need it -#until u1db adds it to its release as a dep. -oauth +# leap deps -- bump me! +leap.common +# XXX -- fix me! +# oauth is not strictly needed by us, but we need it until u1db adds it to its +# release as a dep. +oauth diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py index c5c4b97f..1e52e3a7 100644 --- a/common/src/leap/soledad/common/__init__.py +++ b/common/src/leap/soledad/common/__init__.py @@ -34,45 +34,17 @@ USER_DB_PREFIX = 'user-' # Global functions # -# we want to use leap.common.check.leap_assert in case it is available, -# because it also logs in a way other parts of leap can access log messages. - -try: - from leap.common.check import leap_assert as soledad_assert - -except ImportError: - - def soledad_assert(condition, message): - """ - Asserts the condition and displays the message if that's not - met. - - @param condition: condition to check - @type condition: bool - @param message: message to display if the condition isn't met - @type message: str - """ - assert condition, message - -try: - from leap.common.check import leap_assert_type as soledad_assert_type - -except ImportError: - - def soledad_assert_type(var, expectedType): - """ - Helper assert check for a variable's expected type - - @param var: variable to check - @type var: any - @param expectedType: type to check agains - @type expectedType: type - """ - soledad_assert(isinstance(var, expectedType), - "Expected type %r instead of %r" % - (expectedType, type(var))) +from leap.common.check import leap_assert as soledad_assert +from leap.common.check import leap_assert_type as soledad_assert_type from ._version import get_versions __version__ = get_versions()['version'] del get_versions + + +__all__ = [ + "soledad_assert", + "soledad_assert_type", + "__version__", +] diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index 89ec52e7..c65ee4f5 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -9,12 +9,7 @@ twisted # leap deps -- bump me! leap.soledad.common>=0.6.0 -# -# Things yet to fix: -# - -# oauth is not strictly needed by us, but we need it -# until u1db adds it to its release as a dep. - +# XXX -- fix me! +# oauth is not strictly needed by us, but we need it until u1db adds it to its +# release as a dep. oauth - -- cgit v1.2.3 From 6f598ff5e2437ae4f966c24cb211d37f6941dffa Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 19 Mar 2015 11:18:47 -0400 Subject: [docs] add git commit template to repo because in OCD we trust. --- docs/leap-commit-template | 7 ++++++ docs/leap-commit-template.README | 47 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 docs/leap-commit-template create mode 100644 docs/leap-commit-template.README diff --git a/docs/leap-commit-template b/docs/leap-commit-template new file mode 100644 index 00000000..8a5c7cd0 --- /dev/null +++ b/docs/leap-commit-template @@ -0,0 +1,7 @@ +[bug|feat|docs|style|refactor|test|pkg|i18n] ... +... + +- Resolves: #XYZ +- Related: #XYZ +- Documentation: #XYZ +- Releases: XYZ diff --git a/docs/leap-commit-template.README b/docs/leap-commit-template.README new file mode 100644 index 00000000..ce8809e7 --- /dev/null +++ b/docs/leap-commit-template.README @@ -0,0 +1,47 @@ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +HOW TO USE THIS TEMPLATE: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Run `git config commit.template docs/leap-commit-template` or +edit the .git/config for this project and add +`template = docs/leap-commit-template` +under the [commit] block + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +COMMIT TEMPLATE FORMAT EXPLAINED +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +[type] + + +