diff options
| author | drebs <drebs@leap.se> | 2016-11-10 23:50:30 -0200 |
|---|---|---|
| committer | drebs <drebs@leap.se> | 2016-11-10 23:50:30 -0200 |
| commit | c1950b41e0995b0213227bd0ce2c633f312037dc (patch) | |
| tree | 7c1fde54442fefd3553d33b3fe5a2ec454e0196b /common/src/leap | |
| parent | 507e284773d9c4954225635741f275c5d327e2a9 (diff) | |
| parent | 6b23b3f3215f2443aa3e790559b63a41b3040072 (diff) | |
Merge tag '0.8.1'
0.8.1
Diffstat (limited to 'common/src/leap')
62 files changed, 4761 insertions, 24021 deletions
diff --git a/common/src/leap/soledad/common/README.txt b/common/src/leap/soledad/common/README.txt index 106efb5e..38b9858e 100644 --- a/common/src/leap/soledad/common/README.txt +++ b/common/src/leap/soledad/common/README.txt @@ -3,10 +3,10 @@ Soledad common package This package contains Soledad bits used by both server and client. -Couch U1DB Backend +Couch L2DB Backend ------------------ -U1DB backends rely on some atomic operations that modify documents contents +L2DB backends rely on some atomic operations that modify documents contents and metadata (conflicts, transaction ids and indexes). The only atomic operation in Couch is a document put, so every u1db atomic operation has to be mapped to a couch document put. diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py index d7f6929c..1ba6ab89 100644 --- a/common/src/leap/soledad/common/__init__.py +++ b/common/src/leap/soledad/common/__init__.py @@ -47,7 +47,3 @@ __all__ = [ "soledad_assert_type", "__version__", ] - -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py index 53426fb5..f4f48f86 100644 --- a/common/src/leap/soledad/common/backend.py +++ b/common/src/leap/soledad/common/backend.py @@ -16,27 +16,28 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -"""A U1DB generic backend.""" +"""A L2DB generic backend.""" +import functools -from u1db import vectorclock -from u1db.errors import ( +from leap.soledad.common.document import ServerDocument +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db.errors import ( RevisionConflict, InvalidDocId, ConflictedDoc, DocumentDoesNotExist, DocumentAlreadyDeleted, ) -from u1db.backends import CommonBackend -from u1db.backends import CommonSyncTarget -from leap.soledad.common.document import ServerDocument +from leap.soledad.common.l2db.backends import CommonBackend +from leap.soledad.common.l2db.backends import CommonSyncTarget class SoledadBackend(CommonBackend): BATCH_SUPPORT = False """ - A U1DB backend implementation. + A L2DB backend implementation. """ def __init__(self, database, replica_uid=None): @@ -438,9 +439,8 @@ class SoledadBackend(CommonBackend): generation. :type other_transaction_id: str """ - function = self._set_replica_gen_and_trans_id args = [other_replica_uid, other_generation, other_transaction_id] - callback = lambda: function(*args) + callback = functools.partial(self._set_replica_gen_and_trans_id, *args) if self.batching: self.after_batch_callbacks['set_source_info'] = callback else: diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py index 18ed8a19..523a50a0 100644 --- a/common/src/leap/soledad/common/couch/__init__.py +++ b/common/src/leap/soledad/common/couch/__init__.py @@ -24,6 +24,7 @@ import re import uuid import binascii import time +import functools from StringIO import StringIO @@ -41,12 +42,12 @@ from couchdb.http import ( urljoin as couch_urljoin, Resource, ) -from u1db.errors import ( +from leap.soledad.common.l2db.errors import ( DatabaseDoesNotExist, InvalidGeneration, RevisionConflict, ) -from u1db.remote import http_app +from leap.soledad.common.l2db.remote import http_app from leap.soledad.common import ddocs @@ -340,7 +341,8 @@ class CouchDatabase(object): # This will not be needed when/if we switch from python-couchdb to # paisley. time.strptime('Mar 8 1917', '%b %d %Y') - get_one = lambda doc_id: self.get_doc(doc_id, check_for_conflicts) + get_one = functools.partial( + self.get_doc, check_for_conflicts=check_for_conflicts) docs = [THREAD_POOL.apply_async(get_one, [doc_id]) for doc_id in doc_ids] for doc in docs: diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py index 4f07c105..9b40a264 100644 --- a/common/src/leap/soledad/common/couch/state.py +++ b/common/src/leap/soledad/common/couch/state.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # state.py -# Copyright (C) 2015 LEAP +# Copyright (C) 2015,2016 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,17 +17,17 @@ """ Server state using CouchDatabase as backend. """ -import re import logging +import re import time from urlparse import urljoin from hashlib import sha512 -from u1db.remote.server_state import ServerState -from leap.soledad.common.command import exec_validated_cmd from leap.soledad.common.couch import CouchDatabase from leap.soledad.common.couch import couch_server -from u1db.errors import Unauthorized +from leap.soledad.common.command import exec_validated_cmd +from leap.soledad.common.l2db.remote.server_state import ServerState +from leap.soledad.common.l2db.errors import Unauthorized logger = logging.getLogger(__name__) diff --git a/common/src/leap/soledad/common/document.py b/common/src/leap/soledad/common/document.py index 9e0c0976..6c26a29f 100644 --- a/common/src/leap/soledad/common/document.py +++ b/common/src/leap/soledad/common/document.py @@ -17,11 +17,11 @@ """ -A Soledad Document is an u1db.Document with lasers. +A Soledad Document is an l2db.Document with lasers. """ -from u1db import Document +from .l2db import Document # diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index 0b6bb4e6..dec871c9 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -20,9 +20,8 @@ Soledad errors. """ - -from u1db import errors -from u1db.remote import http_errors +from .l2db import errors +from .l2db.remote import http_errors def register_exception(cls): @@ -71,67 +70,6 @@ class InvalidAuthTokenError(errors.Unauthorized): # -# LockResource errors -# - -@register_exception -class InvalidTokenError(SoledadError): - - """ - Exception raised when trying to unlock shared database with invalid token. - """ - - wire_description = "unlock unauthorized" - status = 401 - - -@register_exception -class NotLockedError(SoledadError): - - """ - Exception raised when trying to unlock shared database when it is not - locked. - """ - - wire_description = "lock not found" - status = 404 - - -@register_exception -class AlreadyLockedError(SoledadError): - - """ - Exception raised when trying to lock shared database but it is already - locked. - """ - - wire_description = "lock is locked" - status = 403 - - -@register_exception -class LockTimedOutError(SoledadError): - - """ - Exception raised when timing out while trying to lock the shared database. - """ - - wire_description = "lock timed out" - status = 408 - - -@register_exception -class CouldNotObtainLockError(SoledadError): - - """ - Exception raised when timing out while trying to lock the shared database. - """ - - wire_description = "error obtaining lock" - status = 500 - - -# # SoledadBackend errors # u1db error statuses also have to be updated http_errors.ERROR_STATUSES = set( diff --git a/common/src/leap/soledad/common/l2db/__init__.py b/common/src/leap/soledad/common/l2db/__init__.py new file mode 100644 index 00000000..c0bd15fe --- /dev/null +++ b/common/src/leap/soledad/common/l2db/__init__.py @@ -0,0 +1,697 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""L2DB""" + +try: + import simplejson as json +except ImportError: + import json # noqa + +from leap.soledad.common.l2db.errors import InvalidJSON, InvalidContent + +__version_info__ = (13, 9) +__version__ = '.'.join(map(lambda x: '%02d' % x, __version_info__)) + + +def open(path, create, document_factory=None): + """Open a database at the given location. + + Will raise u1db.errors.DatabaseDoesNotExist if create=False and the + database does not already exist. + + :param path: The filesystem path for the database to open. + :param create: True/False, should the database be created if it doesn't + already exist? + :param document_factory: A function that will be called with the same + parameters as Document.__init__. + :return: An instance of Database. + """ + from leap.soledad.common.l2db.backends import sqlite_backend + return sqlite_backend.SQLiteDatabase.open_database( + path, create=create, document_factory=document_factory) + + +# constraints on database names (relevant for remote access, as regex) +DBNAME_CONSTRAINTS = r"[a-zA-Z0-9][a-zA-Z0-9.-]*" + +# constraints on doc ids (as regex) +# (no slashes, and no characters outside the ascii range) +DOC_ID_CONSTRAINTS = r"[a-zA-Z0-9.%_-]+" + + +class Database(object): + """A JSON Document data store. + + This data store can be synchronized with other u1db.Database instances. + """ + + def set_document_factory(self, factory): + """Set the document factory that will be used to create objects to be + returned as documents by the database. + + :param factory: A function that returns an object which at minimum must + satisfy the same interface as does the class DocumentBase. + Subclassing that class is the easiest way to create such + a function. + """ + raise NotImplementedError(self.set_document_factory) + + def set_document_size_limit(self, limit): + """Set the maximum allowed document size for this database. + + :param limit: Maximum allowed document size in bytes. + """ + raise NotImplementedError(self.set_document_size_limit) + + def whats_changed(self, old_generation=0): + """Return a list of documents that have changed since old_generation. + This allows APPS to only store a db generation before going + 'offline', and then when coming back online they can use this + data to update whatever extra data they are storing. + + :param old_generation: The generation of the database in the old + state. + :return: (generation, trans_id, [(doc_id, generation, trans_id),...]) + The current generation of the database, its associated transaction + id, and a list of of changed documents since old_generation, + represented by tuples with for each document its doc_id and the + generation and transaction id corresponding to the last intervening + change and sorted by generation (old changes first) + """ + raise NotImplementedError(self.whats_changed) + + def get_doc(self, doc_id, include_deleted=False): + """Get the JSON string for the given document. + + :param doc_id: The unique document identifier + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise asking for a deleted + document will return None. + :return: a Document object. + """ + raise NotImplementedError(self.get_doc) + + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + """Get the JSON content for many documents. + + :param doc_ids: A list of document identifiers. + :param check_for_conflicts: If set to False, then the conflict check + will be skipped, and 'None' will be returned instead of True/False. + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :return: iterable giving the Document object for each document id + in matching doc_ids order. + """ + raise NotImplementedError(self.get_docs) + + def get_all_docs(self, include_deleted=False): + """Get the JSON content for all documents in the database. + + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :return: (generation, [Document]) + The current generation of the database, followed by a list of all + the documents in the database. + """ + raise NotImplementedError(self.get_all_docs) + + def create_doc(self, content, doc_id=None): + """Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param content: A Python dictionary. + :param doc_id: An optional identifier specifying the document id. + :return: Document + """ + raise NotImplementedError(self.create_doc) + + def create_doc_from_json(self, json, doc_id=None): + """Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param json: The JSON document string + :param doc_id: An optional identifier specifying the document id. + :return: Document + """ + raise NotImplementedError(self.create_doc_from_json) + + def put_doc(self, doc): + """Update a document. + If the document currently has conflicts, put will fail. + If the database specifies a maximum document size and the document + exceeds it, put will fail and raise a DocumentTooBig exception. + + :param doc: A Document with new content. + :return: new_doc_rev - The new revision identifier for the document. + The Document object will also be updated. + """ + raise NotImplementedError(self.put_doc) + + def delete_doc(self, doc): + """Mark a document as deleted. + Will abort if the current revision doesn't match doc.rev. + This will also set doc.content to None. + """ + raise NotImplementedError(self.delete_doc) + + def create_index(self, index_name, *index_expressions): + """Create an named index, which can then be queried for future lookups. + Creating an index which already exists is not an error, and is cheap. + Creating an index which does not match the index_expressions of the + existing index is an error. + Creating an index will block until the expressions have been evaluated + and the index generated. + + :param index_name: A unique name which can be used as a key prefix + :param index_expressions: index expressions defining the index + information. + + Examples: + + "fieldname", or "fieldname.subfieldname" to index alphabetically + sorted on the contents of a field. + + "number(fieldname, width)", "lower(fieldname)" + """ + raise NotImplementedError(self.create_index) + + def delete_index(self, index_name): + """Remove a named index. + + :param index_name: The name of the index we are removing + """ + raise NotImplementedError(self.delete_index) + + def list_indexes(self): + """List the definitions of all known indexes. + + :return: A list of [('index-name', ['field', 'field2'])] definitions. + """ + raise NotImplementedError(self.list_indexes) + + def get_from_index(self, index_name, *key_values): + """Return documents that match the keys supplied. + + You must supply exactly the same number of values as have been defined + in the index. It is possible to do a prefix match by using '*' to + indicate a wildcard match. You can only supply '*' to trailing entries, + (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) + It is also possible to append a '*' to the last supplied value (eg + 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :return: List of [Document] + """ + raise NotImplementedError(self.get_from_index) + + def get_range_from_index(self, index_name, start_value, end_value): + """Return documents that fall within the specified range. + + Both ends of the range are inclusive. For both start_value and + end_value, one must supply exactly the same number of values as have + been defined in the index, or pass None. In case of a single column + index, a string is accepted as an alternative for a tuple with a single + value. It is possible to do a prefix match by using '*' to indicate + a wildcard match. You can only supply '*' to trailing entries, (eg + 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also + possible to append a '*' to the last supplied value (eg 'val*', '*', + '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :param start_values: tuples of values that define the lower bound of + the range. eg, if you have an index with 3 fields then you would + have: (val1, val2, val3) + :param end_values: tuples of values that define the upper bound of the + range. eg, if you have an index with 3 fields then you would have: + (val1, val2, val3) + :return: List of [Document] + """ + raise NotImplementedError(self.get_range_from_index) + + def get_index_keys(self, index_name): + """Return all keys under which documents are indexed in this index. + + :param index_name: The index to query + :return: [] A list of tuples of indexed keys. + """ + raise NotImplementedError(self.get_index_keys) + + def get_doc_conflicts(self, doc_id): + """Get the list of conflicts for the given document. + + The order of the conflicts is such that the first entry is the value + that would be returned by "get_doc". + + :return: [doc] A list of the Document entries that are conflicted. + """ + raise NotImplementedError(self.get_doc_conflicts) + + def resolve_doc(self, doc, conflicted_doc_revs): + """Mark a document as no longer conflicted. + + We take the list of revisions that the client knows about that it is + superseding. This may be a different list from the actual current + conflicts, in which case only those are removed as conflicted. This + may fail if the conflict list is significantly different from the + supplied information. (sync could have happened in the background from + the time you GET_DOC_CONFLICTS until the point where you RESOLVE) + + :param doc: A Document with the new content to be inserted. + :param conflicted_doc_revs: A list of revisions that the new content + supersedes. + """ + raise NotImplementedError(self.resolve_doc) + + def get_sync_target(self): + """Return a SyncTarget object, for another u1db to synchronize with. + + :return: An instance of SyncTarget. + """ + raise NotImplementedError(self.get_sync_target) + + def close(self): + """Release any resources associated with this database.""" + raise NotImplementedError(self.close) + + def sync(self, url, creds=None, autocreate=True): + """Synchronize documents with remote replica exposed at url. + + :param url: the url of the target replica to sync with. + :param creds: optional dictionary giving credentials + to authorize the operation with the server. For using OAuth + the form of creds is: + {'oauth': { + 'consumer_key': ..., + 'consumer_secret': ..., + 'token_key': ..., + 'token_secret': ... + }} + :param autocreate: ask the target to create the db if non-existent. + :return: local_gen_before_sync The local generation before the + synchronisation was performed. This is useful to pass into + whatschanged, if an application wants to know which documents were + affected by a synchronisation. + """ + from u1db.sync import Synchronizer + from u1db.remote.http_target import HTTPSyncTarget + return Synchronizer(self, HTTPSyncTarget(url, creds=creds)).sync( + autocreate=autocreate) + + def _get_replica_gen_and_trans_id(self, other_replica_uid): + """Return the last known generation and transaction id for the other db + replica. + + When you do a synchronization with another replica, the Database keeps + track of what generation the other database replica was at, and what + the associated transaction id was. This is used to determine what data + needs to be sent, and if two databases are claiming to be the same + replica. + + :param other_replica_uid: The identifier for the other replica. + :return: (gen, trans_id) The generation and transaction id we + encountered during synchronization. If we've never synchronized + with the replica, this is (0, ''). + """ + raise NotImplementedError(self._get_replica_gen_and_trans_id) + + def _set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, other_transaction_id): + """Set the last-known generation and transaction id for the other + database replica. + + We have just performed some synchronization, and we want to track what + generation the other replica was at. See also + _get_replica_gen_and_trans_id. + :param other_replica_uid: The U1DB identifier for the other replica. + :param other_generation: The generation number for the other replica. + :param other_transaction_id: The transaction id associated with the + generation. + """ + raise NotImplementedError(self._set_replica_gen_and_trans_id) + + def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen, + replica_trans_id=''): + """Insert/update document into the database with a given revision. + + This api is used during synchronization operations. + + If a document would conflict and save_conflict is set to True, the + content will be selected as the 'current' content for doc.doc_id, + even though doc.rev doesn't supersede the currently stored revision. + The currently stored document will be added to the list of conflict + alternatives for the given doc_id. + + This forces the new content to be 'current' so that we get convergence + after synchronizing, even if people don't resolve conflicts. Users can + then notice that their content is out of date, update it, and + synchronize again. (The alternative is that users could synchronize and + think the data has propagated, but their local copy looks fine, and the + remote copy is never updated again.) + + :param doc: A Document object + :param save_conflict: If this document is a conflict, do you want to + save it as a conflict, or just ignore it. + :param replica_uid: A unique replica identifier. + :param replica_gen: The generation of the replica corresponding to the + this document. The replica arguments are optional, but are used + during synchronization. + :param replica_trans_id: The transaction_id associated with the + generation. + :return: (state, at_gen) - If we don't have doc_id already, + or if doc_rev supersedes the existing document revision, + then the content will be inserted, and state is 'inserted'. + If doc_rev is less than or equal to the existing revision, + then the put is ignored and state is respecitvely 'superseded' + or 'converged'. + If doc_rev is not strictly superseded or supersedes, then + state is 'conflicted'. The document will not be inserted if + save_conflict is False. + For 'inserted' or 'converged', at_gen is the insertion/current + generation. + """ + raise NotImplementedError(self._put_doc_if_newer) + + +class DocumentBase(object): + """Container for handling a single document. + + :ivar doc_id: Unique identifier for this document. + :ivar rev: The revision identifier of the document. + :ivar json_string: The JSON string for this document. + :ivar has_conflicts: Boolean indicating if this document has conflicts + """ + + def __init__(self, doc_id, rev, json_string, has_conflicts=False): + self.doc_id = doc_id + self.rev = rev + if json_string is not None: + try: + value = json.loads(json_string) + except ValueError: + raise InvalidJSON + if not isinstance(value, dict): + raise InvalidJSON + self._json = json_string + self.has_conflicts = has_conflicts + + def same_content_as(self, other): + """Compare the content of two documents.""" + if self._json: + c1 = json.loads(self._json) + else: + c1 = None + if other._json: + c2 = json.loads(other._json) + else: + c2 = None + return c1 == c2 + + def __repr__(self): + if self.has_conflicts: + extra = ', conflicted' + else: + extra = '' + return '%s(%s, %s%s, %r)' % (self.__class__.__name__, self.doc_id, + self.rev, extra, self.get_json()) + + def __hash__(self): + raise NotImplementedError(self.__hash__) + + def __eq__(self, other): + if not isinstance(other, Document): + return NotImplemented + return ( + self.doc_id == other.doc_id and self.rev == other.rev and + self.same_content_as(other) and self.has_conflicts == + other.has_conflicts) + + def __lt__(self, other): + """This is meant for testing, not part of the official api. + + It is implemented so that sorted([Document, Document]) can be used. + It doesn't imply that users would want their documents to be sorted in + this order. + """ + # Since this is just for testing, we don't worry about comparing + # against things that aren't a Document. + return ((self.doc_id, self.rev, self.get_json()) < + (other.doc_id, other.rev, other.get_json())) + + def get_json(self): + """Get the json serialization of this document.""" + if self._json is not None: + return self._json + return None + + def get_size(self): + """Calculate the total size of the document.""" + size = 0 + json = self.get_json() + if json: + size += len(json) + if self.rev: + size += len(self.rev) + if self.doc_id: + size += len(self.doc_id) + return size + + def set_json(self, json_string): + """Set the json serialization of this document.""" + if json_string is not None: + try: + value = json.loads(json_string) + except ValueError: + raise InvalidJSON + if not isinstance(value, dict): + raise InvalidJSON + self._json = json_string + + def make_tombstone(self): + """Make this document into a tombstone.""" + self._json = None + + def is_tombstone(self): + """Return True if the document is a tombstone, False otherwise.""" + if self._json is not None: + return False + return True + + +class Document(DocumentBase): + """Container for handling a single document. + + :ivar doc_id: Unique identifier for this document. + :ivar rev: The revision identifier of the document. + :ivar json: The JSON string for this document. + :ivar has_conflicts: Boolean indicating if this document has conflicts + """ + + # The following part of the API is optional: no implementation is forced to + # have it but if the language supports dictionaries/hashtables, it makes + # Documents a lot more user friendly. + + def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False): + # TODO: We convert the json in the superclass to check its validity so + # we might as well set _content here directly since the price is + # already being paid. + super(Document, self).__init__(doc_id, rev, json, has_conflicts) + self._content = None + + def same_content_as(self, other): + """Compare the content of two documents.""" + if self._json: + c1 = json.loads(self._json) + else: + c1 = self._content + if other._json: + c2 = json.loads(other._json) + else: + c2 = other._content + return c1 == c2 + + def get_json(self): + """Get the json serialization of this document.""" + json_string = super(Document, self).get_json() + if json_string is not None: + return json_string + if self._content is not None: + return json.dumps(self._content) + return None + + def set_json(self, json): + """Set the json serialization of this document.""" + self._content = None + super(Document, self).set_json(json) + + def make_tombstone(self): + """Make this document into a tombstone.""" + self._content = None + super(Document, self).make_tombstone() + + def is_tombstone(self): + """Return True if the document is a tombstone, False otherwise.""" + if self._content is not None: + return False + return super(Document, self).is_tombstone() + + def _get_content(self): + """Get the dictionary representing this document.""" + if self._json is not None: + self._content = json.loads(self._json) + self._json = None + if self._content is not None: + return self._content + return None + + def _set_content(self, content): + """Set the dictionary representing this document.""" + try: + tmp = json.dumps(content) + except TypeError: + raise InvalidContent( + "Can not be converted to JSON: %r" % (content,)) + if not tmp.startswith('{'): + raise InvalidContent( + "Can not be converted to a JSON object: %r." % (content,)) + # We might as well store the JSON at this point since we did the work + # of encoding it, and it doesn't lose any information. + self._json = tmp + self._content = None + + content = property( + _get_content, _set_content, doc="Content of the Document.") + + # End of optional part. + + +class SyncTarget(object): + """Functionality for using a Database as a synchronization target.""" + + def get_sync_info(self, source_replica_uid): + """Return information about known state. + + Return the replica_uid and the current database generation of this + database, and the last-seen database generation for source_replica_uid + + :param source_replica_uid: Another replica which we might have + synchronized with in the past. + :return: (target_replica_uid, target_replica_generation, + target_trans_id, source_replica_last_known_generation, + source_replica_last_known_transaction_id) + """ + raise NotImplementedError(self.get_sync_info) + + def record_sync_info(self, source_replica_uid, source_replica_generation, + source_replica_transaction_id): + """Record tip information for another replica. + + After sync_exchange has been processed, the caller will have + received new content from this replica. This call allows the + source replica instigating the sync to inform us what their + generation became after applying the documents we returned. + + This is used to allow future sync operations to not need to repeat data + that we just talked about. It also means that if this is called at the + wrong time, there can be database records that will never be + synchronized. + + :param source_replica_uid: The identifier for the source replica. + :param source_replica_generation: + The database generation for the source replica. + :param source_replica_transaction_id: The transaction id associated + with the source replica generation. + """ + raise NotImplementedError(self.record_sync_info) + + def sync_exchange(self, docs_by_generation, source_replica_uid, + last_known_generation, last_known_trans_id, + return_doc_cb, ensure_callback=None): + """Incorporate the documents sent from the source replica. + + This is not meant to be called by client code directly, but is used as + part of sync(). + + This adds docs to the local store, and determines documents that need + to be returned to the source replica. + + Documents must be supplied in docs_by_generation paired with + the generation of their latest change in order from the oldest + change to the newest, that means from the oldest generation to + the newest. + + Documents are also returned paired with the generation of + their latest change in order from the oldest change to the + newest. + + :param docs_by_generation: A list of [(Document, generation, + transaction_id)] tuples indicating documents which should be + updated on this replica paired with the generation and transaction + id of their latest change. + :param source_replica_uid: The source replica's identifier + :param last_known_generation: The last generation that the source + replica knows about this target replica + :param last_known_trans_id: The last transaction id that the source + replica knows about this target replica + :param: return_doc_cb(doc, gen): is a callback + used to return documents to the source replica, it will + be invoked in turn with Documents that have changed since + last_known_generation together with the generation of + their last change. + :param: ensure_callback(replica_uid): if set the target may create + the target db if not yet existent, the callback can then + be used to inform of the created db replica uid. + :return: new_generation - After applying docs_by_generation, this is + the current generation for this replica + """ + raise NotImplementedError(self.sync_exchange) + + def _set_trace_hook(self, cb): + """Set a callback that will be invoked to trace database actions. + + The callback will be passed a string indicating the current state, and + the sync target object. Implementations do not have to implement this + api, it is used by the test suite. + + :param cb: A callable that takes cb(state) + """ + raise NotImplementedError(self._set_trace_hook) + + def _set_trace_hook_shallow(self, cb): + """Set a callback that will be invoked to trace database actions. + + Similar to _set_trace_hook, for implementations that don't offer + state changes from the inner working of sync_exchange(). + + :param cb: A callable that takes cb(state) + """ + self._set_trace_hook(cb) diff --git a/common/src/leap/soledad/common/l2db/backends/__init__.py b/common/src/leap/soledad/common/l2db/backends/__init__.py new file mode 100644 index 00000000..922daafd --- /dev/null +++ b/common/src/leap/soledad/common/l2db/backends/__init__.py @@ -0,0 +1,207 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Abstract classes and common implementations for the backends.""" + +import re +try: + import simplejson as json +except ImportError: + import json # noqa +import uuid + +from leap.soledad.common import l2db +from leap.soledad.common.l2db import sync as l2db_sync +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.vectorclock import VectorClockRev + + +check_doc_id_re = re.compile("^" + l2db.DOC_ID_CONSTRAINTS + "$", re.UNICODE) + + +class CommonSyncTarget(l2db_sync.LocalSyncTarget): + pass + + +class CommonBackend(l2db.Database): + + document_size_limit = 0 + + def _allocate_doc_id(self): + """Generate a unique identifier for this document.""" + return 'D-' + uuid.uuid4().hex # 'D-' stands for document + + def _allocate_transaction_id(self): + return 'T-' + uuid.uuid4().hex # 'T-' stands for transaction + + def _allocate_doc_rev(self, old_doc_rev): + vcr = VectorClockRev(old_doc_rev) + vcr.increment(self._replica_uid) + return vcr.as_str() + + def _check_doc_id(self, doc_id): + if not check_doc_id_re.match(doc_id): + raise errors.InvalidDocId() + + def _check_doc_size(self, doc): + if not self.document_size_limit: + return + if doc.get_size() > self.document_size_limit: + raise errors.DocumentTooBig + + def _get_generation(self): + """Return the current generation. + + """ + raise NotImplementedError(self._get_generation) + + def _get_generation_info(self): + """Return the current generation and transaction id. + + """ + raise NotImplementedError(self._get_generation_info) + + def _get_doc(self, doc_id, check_for_conflicts=False): + """Extract the document from storage. + + This can return None if the document doesn't exist. + """ + raise NotImplementedError(self._get_doc) + + def _has_conflicts(self, doc_id): + """Return True if the doc has conflicts, False otherwise.""" + raise NotImplementedError(self._has_conflicts) + + def create_doc(self, content, doc_id=None): + if not isinstance(content, dict): + raise errors.InvalidContent + json_string = json.dumps(content) + return self.create_doc_from_json(json_string, doc_id) + + def create_doc_from_json(self, json, doc_id=None): + if doc_id is None: + doc_id = self._allocate_doc_id() + doc = self._factory(doc_id, None, json) + self.put_doc(doc) + return doc + + def _get_transaction_log(self): + """This is only for the test suite, it is not part of the api.""" + raise NotImplementedError(self._get_transaction_log) + + def _put_and_update_indexes(self, doc_id, old_doc, new_rev, content): + raise NotImplementedError(self._put_and_update_indexes) + + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + for doc_id in doc_ids: + doc = self._get_doc( + doc_id, check_for_conflicts=check_for_conflicts) + if doc.is_tombstone() and not include_deleted: + continue + yield doc + + def _get_trans_id_for_gen(self, generation): + """Get the transaction id corresponding to a particular generation. + + Raises an InvalidGeneration when the generation does not exist. + + """ + raise NotImplementedError(self._get_trans_id_for_gen) + + def validate_gen_and_trans_id(self, generation, trans_id): + """Validate the generation and transaction id. + + Raises an InvalidGeneration when the generation does not exist, and an + InvalidTransactionId when it does but with a different transaction id. + + """ + if generation == 0: + return + known_trans_id = self._get_trans_id_for_gen(generation) + if known_trans_id != trans_id: + raise errors.InvalidTransactionId + + def _validate_source(self, other_replica_uid, other_generation, + other_transaction_id): + """Validate the new generation and transaction id. + + other_generation must be greater than what we have stored for this + replica, *or* it must be the same and the transaction_id must be the + same as well. + """ + (old_generation, + old_transaction_id) = self._get_replica_gen_and_trans_id( + other_replica_uid) + if other_generation < old_generation: + raise errors.InvalidGeneration + if other_generation > old_generation: + return + if other_transaction_id == old_transaction_id: + return + raise errors.InvalidTransactionId + + def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen, + replica_trans_id=''): + cur_doc = self._get_doc(doc.doc_id) + doc_vcr = VectorClockRev(doc.rev) + if cur_doc is None: + cur_vcr = VectorClockRev(None) + else: + cur_vcr = VectorClockRev(cur_doc.rev) + self._validate_source(replica_uid, replica_gen, replica_trans_id) + if doc_vcr.is_newer(cur_vcr): + rev = doc.rev + self._prune_conflicts(doc, doc_vcr) + if doc.rev != rev: + # conflicts have been autoresolved + state = 'superseded' + else: + state = 'inserted' + self._put_and_update_indexes(cur_doc, doc) + elif doc.rev == cur_doc.rev: + # magical convergence + state = 'converged' + elif cur_vcr.is_newer(doc_vcr): + # Don't add this to seen_ids, because we have something newer, + # so we should send it back, and we should not generate a + # conflict + state = 'superseded' + elif cur_doc.same_content_as(doc): + # the documents have been edited to the same thing at both ends + doc_vcr.maximize(cur_vcr) + doc_vcr.increment(self._replica_uid) + doc.rev = doc_vcr.as_str() + self._put_and_update_indexes(cur_doc, doc) + state = 'superseded' + else: + state = 'conflicted' + if save_conflict: + self._force_doc_sync_conflict(doc) + if replica_uid is not None and replica_gen is not None: + self._do_set_replica_gen_and_trans_id( + replica_uid, replica_gen, replica_trans_id) + return state, self._get_generation() + + def _ensure_maximal_rev(self, cur_rev, extra_revs): + vcr = VectorClockRev(cur_rev) + for rev in extra_revs: + vcr.maximize(VectorClockRev(rev)) + vcr.increment(self._replica_uid) + return vcr.as_str() + + def set_document_size_limit(self, limit): + self.document_size_limit = limit diff --git a/common/src/leap/soledad/common/l2db/backends/dbschema.sql b/common/src/leap/soledad/common/l2db/backends/dbschema.sql new file mode 100644 index 00000000..ae027fc5 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/backends/dbschema.sql @@ -0,0 +1,42 @@ +-- Database schema +CREATE TABLE transaction_log ( + generation INTEGER PRIMARY KEY AUTOINCREMENT, + doc_id TEXT NOT NULL, + transaction_id TEXT NOT NULL +); +CREATE TABLE document ( + doc_id TEXT PRIMARY KEY, + doc_rev TEXT NOT NULL, + content TEXT +); +CREATE TABLE document_fields ( + doc_id TEXT NOT NULL, + field_name TEXT NOT NULL, + value TEXT +); +CREATE INDEX document_fields_field_value_doc_idx + ON document_fields(field_name, value, doc_id); + +CREATE TABLE sync_log ( + replica_uid TEXT PRIMARY KEY, + known_generation INTEGER, + known_transaction_id TEXT +); +CREATE TABLE conflicts ( + doc_id TEXT, + doc_rev TEXT, + content TEXT, + CONSTRAINT conflicts_pkey PRIMARY KEY (doc_id, doc_rev) +); +CREATE TABLE index_definitions ( + name TEXT, + offset INT, + field TEXT, + CONSTRAINT index_definitions_pkey PRIMARY KEY (name, offset) +); +create index index_definitions_field on index_definitions(field); +CREATE TABLE u1db_config ( + name TEXT PRIMARY KEY, + value TEXT +); +INSERT INTO u1db_config VALUES ('sql_schema', '0'); diff --git a/common/src/leap/soledad/common/l2db/backends/inmemory.py b/common/src/leap/soledad/common/l2db/backends/inmemory.py new file mode 100644 index 00000000..06a934a6 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/backends/inmemory.py @@ -0,0 +1,469 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""The in-memory Database class for U1DB.""" + +try: + import simplejson as json +except ImportError: + import json # noqa + +from leap.soledad.common.l2db import ( + Document, errors, + query_parser, vectorclock) +from leap.soledad.common.l2db.backends import CommonBackend, CommonSyncTarget + + +def get_prefix(value): + key_prefix = '\x01'.join(value) + return key_prefix.rstrip('*') + + +class InMemoryDatabase(CommonBackend): + """A database that only stores the data internally.""" + + def __init__(self, replica_uid, document_factory=None): + self._transaction_log = [] + self._docs = {} + # Map from doc_id => [(doc_rev, doc)] conflicts beyond 'winner' + self._conflicts = {} + self._other_generations = {} + self._indexes = {} + self._replica_uid = replica_uid + self._factory = document_factory or Document + + def _set_replica_uid(self, replica_uid): + """Force the replica_uid to be set.""" + self._replica_uid = replica_uid + + def set_document_factory(self, factory): + self._factory = factory + + def close(self): + # This is a no-op, We don't want to free the data because one client + # may be closing it, while another wants to inspect the results. + pass + + def _get_replica_gen_and_trans_id(self, other_replica_uid): + return self._other_generations.get(other_replica_uid, (0, '')) + + def _set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, other_transaction_id): + self._do_set_replica_gen_and_trans_id( + other_replica_uid, other_generation, other_transaction_id) + + def _do_set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, + other_transaction_id): + # TODO: to handle race conditions, we may want to check if the current + # value is greater than this new value. + self._other_generations[other_replica_uid] = (other_generation, + other_transaction_id) + + def get_sync_target(self): + return InMemorySyncTarget(self) + + def _get_transaction_log(self): + # snapshot! + return self._transaction_log[:] + + def _get_generation(self): + return len(self._transaction_log) + + def _get_generation_info(self): + if not self._transaction_log: + return 0, '' + return len(self._transaction_log), self._transaction_log[-1][1] + + def _get_trans_id_for_gen(self, generation): + if generation == 0: + return '' + if generation > len(self._transaction_log): + raise errors.InvalidGeneration + return self._transaction_log[generation - 1][1] + + def put_doc(self, doc): + if doc.doc_id is None: + raise errors.InvalidDocId() + self._check_doc_id(doc.doc_id) + self._check_doc_size(doc) + old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) + if old_doc and old_doc.has_conflicts: + raise errors.ConflictedDoc() + if old_doc and doc.rev is None and old_doc.is_tombstone(): + new_rev = self._allocate_doc_rev(old_doc.rev) + else: + if old_doc is not None: + if old_doc.rev != doc.rev: + raise errors.RevisionConflict() + else: + if doc.rev is not None: + raise errors.RevisionConflict() + new_rev = self._allocate_doc_rev(doc.rev) + doc.rev = new_rev + self._put_and_update_indexes(old_doc, doc) + return new_rev + + def _put_and_update_indexes(self, old_doc, doc): + for index in self._indexes.itervalues(): + if old_doc is not None and not old_doc.is_tombstone(): + index.remove_json(old_doc.doc_id, old_doc.get_json()) + if not doc.is_tombstone(): + index.add_json(doc.doc_id, doc.get_json()) + trans_id = self._allocate_transaction_id() + self._docs[doc.doc_id] = (doc.rev, doc.get_json()) + self._transaction_log.append((doc.doc_id, trans_id)) + + def _get_doc(self, doc_id, check_for_conflicts=False): + try: + doc_rev, content = self._docs[doc_id] + except KeyError: + return None + doc = self._factory(doc_id, doc_rev, content) + if check_for_conflicts: + doc.has_conflicts = (doc.doc_id in self._conflicts) + return doc + + def _has_conflicts(self, doc_id): + return doc_id in self._conflicts + + def get_doc(self, doc_id, include_deleted=False): + doc = self._get_doc(doc_id, check_for_conflicts=True) + if doc is None: + return None + if doc.is_tombstone() and not include_deleted: + return None + return doc + + def get_all_docs(self, include_deleted=False): + """Return all documents in the database.""" + generation = self._get_generation() + results = [] + for doc_id, (doc_rev, content) in self._docs.items(): + if content is None and not include_deleted: + continue + doc = self._factory(doc_id, doc_rev, content) + doc.has_conflicts = self._has_conflicts(doc_id) + results.append(doc) + return (generation, results) + + def get_doc_conflicts(self, doc_id): + if doc_id not in self._conflicts: + return [] + result = [self._get_doc(doc_id)] + result[0].has_conflicts = True + result.extend([self._factory(doc_id, rev, content) + for rev, content in self._conflicts[doc_id]]) + return result + + def _replace_conflicts(self, doc, conflicts): + if not conflicts: + del self._conflicts[doc.doc_id] + else: + self._conflicts[doc.doc_id] = conflicts + doc.has_conflicts = bool(conflicts) + + def _prune_conflicts(self, doc, doc_vcr): + if self._has_conflicts(doc.doc_id): + autoresolved = False + remaining_conflicts = [] + cur_conflicts = self._conflicts[doc.doc_id] + for c_rev, c_doc in cur_conflicts: + c_vcr = vectorclock.VectorClockRev(c_rev) + if doc_vcr.is_newer(c_vcr): + continue + if doc.same_content_as(Document(doc.doc_id, c_rev, c_doc)): + doc_vcr.maximize(c_vcr) + autoresolved = True + continue + remaining_conflicts.append((c_rev, c_doc)) + if autoresolved: + doc_vcr.increment(self._replica_uid) + doc.rev = doc_vcr.as_str() + self._replace_conflicts(doc, remaining_conflicts) + + def resolve_doc(self, doc, conflicted_doc_revs): + cur_doc = self._get_doc(doc.doc_id) + if cur_doc is None: + cur_rev = None + else: + cur_rev = cur_doc.rev + new_rev = self._ensure_maximal_rev(cur_rev, conflicted_doc_revs) + superseded_revs = set(conflicted_doc_revs) + remaining_conflicts = [] + cur_conflicts = self._conflicts[doc.doc_id] + for c_rev, c_doc in cur_conflicts: + if c_rev in superseded_revs: + continue + remaining_conflicts.append((c_rev, c_doc)) + doc.rev = new_rev + if cur_rev in superseded_revs: + self._put_and_update_indexes(cur_doc, doc) + else: + remaining_conflicts.append((new_rev, doc.get_json())) + self._replace_conflicts(doc, remaining_conflicts) + + def delete_doc(self, doc): + if doc.doc_id not in self._docs: + raise errors.DocumentDoesNotExist + if self._docs[doc.doc_id][1] in ('null', None): + raise errors.DocumentAlreadyDeleted + doc.make_tombstone() + self.put_doc(doc) + + def create_index(self, index_name, *index_expressions): + if index_name in self._indexes: + if self._indexes[index_name]._definition == list( + index_expressions): + return + raise errors.IndexNameTakenError + index = InMemoryIndex(index_name, list(index_expressions)) + for doc_id, (doc_rev, doc) in self._docs.iteritems(): + if doc is not None: + index.add_json(doc_id, doc) + self._indexes[index_name] = index + + def delete_index(self, index_name): + try: + del self._indexes[index_name] + except KeyError: + pass + + def list_indexes(self): + definitions = [] + for idx in self._indexes.itervalues(): + definitions.append((idx._name, idx._definition)) + return definitions + + def get_from_index(self, index_name, *key_values): + try: + index = self._indexes[index_name] + except KeyError: + raise errors.IndexDoesNotExist + doc_ids = index.lookup(key_values) + result = [] + for doc_id in doc_ids: + result.append(self._get_doc(doc_id, check_for_conflicts=True)) + return result + + def get_range_from_index(self, index_name, start_value=None, + end_value=None): + """Return all documents with key values in the specified range.""" + try: + index = self._indexes[index_name] + except KeyError: + raise errors.IndexDoesNotExist + if isinstance(start_value, basestring): + start_value = (start_value,) + if isinstance(end_value, basestring): + end_value = (end_value,) + doc_ids = index.lookup_range(start_value, end_value) + result = [] + for doc_id in doc_ids: + result.append(self._get_doc(doc_id, check_for_conflicts=True)) + return result + + def get_index_keys(self, index_name): + try: + index = self._indexes[index_name] + except KeyError: + raise errors.IndexDoesNotExist + keys = index.keys() + # XXX inefficiency warning + return list(set([tuple(key.split('\x01')) for key in keys])) + + def whats_changed(self, old_generation=0): + changes = [] + relevant_tail = self._transaction_log[old_generation:] + # We don't use len(self._transaction_log) because _transaction_log may + # get mutated by a concurrent operation. + cur_generation = old_generation + len(relevant_tail) + last_trans_id = '' + if relevant_tail: + last_trans_id = relevant_tail[-1][1] + elif self._transaction_log: + last_trans_id = self._transaction_log[-1][1] + seen = set() + generation = cur_generation + for doc_id, trans_id in reversed(relevant_tail): + if doc_id not in seen: + changes.append((doc_id, generation, trans_id)) + seen.add(doc_id) + generation -= 1 + changes.reverse() + return (cur_generation, last_trans_id, changes) + + def _force_doc_sync_conflict(self, doc): + my_doc = self._get_doc(doc.doc_id) + self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev)) + self._conflicts.setdefault(doc.doc_id, []).append( + (my_doc.rev, my_doc.get_json())) + doc.has_conflicts = True + self._put_and_update_indexes(my_doc, doc) + + +class InMemoryIndex(object): + """Interface for managing an Index.""" + + def __init__(self, index_name, index_definition): + self._name = index_name + self._definition = index_definition + self._values = {} + parser = query_parser.Parser() + self._getters = parser.parse_all(self._definition) + + def evaluate_json(self, doc): + """Determine the 'key' after applying this index to the doc.""" + raw = json.loads(doc) + return self.evaluate(raw) + + def evaluate(self, obj): + """Evaluate a dict object, applying this definition.""" + all_rows = [[]] + for getter in self._getters: + new_rows = [] + keys = getter.get(obj) + if not keys: + return [] + for key in keys: + new_rows.extend([row + [key] for row in all_rows]) + all_rows = new_rows + all_rows = ['\x01'.join(row) for row in all_rows] + return all_rows + + def add_json(self, doc_id, doc): + """Add this json doc to the index.""" + keys = self.evaluate_json(doc) + if not keys: + return + for key in keys: + self._values.setdefault(key, []).append(doc_id) + + def remove_json(self, doc_id, doc): + """Remove this json doc from the index.""" + keys = self.evaluate_json(doc) + if keys: + for key in keys: + doc_ids = self._values[key] + doc_ids.remove(doc_id) + if not doc_ids: + del self._values[key] + + def _find_non_wildcards(self, values): + """Check if this should be a wildcard match. + + Further, this will raise an exception if the syntax is improperly + defined. + + :return: The offset of the last value we need to match against. + """ + if len(values) != len(self._definition): + raise errors.InvalidValueForIndex() + is_wildcard = False + last = 0 + for idx, val in enumerate(values): + if val.endswith('*'): + if val != '*': + # We have an 'x*' style wildcard + if is_wildcard: + # We were already in wildcard mode, so this is invalid + raise errors.InvalidGlobbing + last = idx + 1 + is_wildcard = True + else: + if is_wildcard: + # We were in wildcard mode, we can't follow that with + # non-wildcard + raise errors.InvalidGlobbing + last = idx + 1 + if not is_wildcard: + return -1 + return last + + def lookup(self, values): + """Find docs that match the values.""" + last = self._find_non_wildcards(values) + if last == -1: + return self._lookup_exact(values) + else: + return self._lookup_prefix(values[:last]) + + def lookup_range(self, start_values, end_values): + """Find docs within the range.""" + # TODO: Wildly inefficient, which is unlikely to be a problem for the + # inmemory implementation. + if start_values: + self._find_non_wildcards(start_values) + start_values = get_prefix(start_values) + if end_values: + if self._find_non_wildcards(end_values) == -1: + exact = True + else: + exact = False + end_values = get_prefix(end_values) + found = [] + for key, doc_ids in sorted(self._values.iteritems()): + if start_values and start_values > key: + continue + if end_values and end_values < key: + if exact: + break + else: + if not key.startswith(end_values): + break + found.extend(doc_ids) + return found + + def keys(self): + """Find the indexed keys.""" + return self._values.keys() + + def _lookup_prefix(self, value): + """Find docs that match the prefix string in values.""" + # TODO: We need a different data structure to make prefix style fast, + # some sort of sorted list would work, but a plain dict doesn't. + key_prefix = get_prefix(value) + all_doc_ids = [] + for key, doc_ids in sorted(self._values.iteritems()): + if key.startswith(key_prefix): + all_doc_ids.extend(doc_ids) + return all_doc_ids + + def _lookup_exact(self, value): + """Find docs that match exactly.""" + key = '\x01'.join(value) + if key in self._values: + return self._values[key] + return () + + +class InMemorySyncTarget(CommonSyncTarget): + + def get_sync_info(self, source_replica_uid): + source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id( + source_replica_uid) + my_gen, my_trans_id = self._db._get_generation_info() + return ( + self._db._replica_uid, my_gen, my_trans_id, source_gen, + source_trans_id) + + def record_sync_info(self, source_replica_uid, source_replica_generation, + source_transaction_id): + if self._trace_hook: + self._trace_hook('record_sync_info') + self._db._set_replica_gen_and_trans_id( + source_replica_uid, source_replica_generation, + source_transaction_id) diff --git a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py new file mode 100644 index 00000000..ba273039 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py @@ -0,0 +1,930 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +""" +A L2DB implementation that uses SQLite as its persistence layer. +""" + +import errno +import os +try: + import simplejson as json +except ImportError: + import json # noqa +from sqlite3 import dbapi2 +import sys +import time +import uuid + +import pkg_resources + +from leap.soledad.common.l2db.backends import CommonBackend, CommonSyncTarget +from leap.soledad.common.l2db import ( + Document, errors, + query_parser, vectorclock) + + +class SQLiteDatabase(CommonBackend): + """A U1DB implementation that uses SQLite as its persistence layer.""" + + _sqlite_registry = {} + + def __init__(self, sqlite_file, document_factory=None): + """Create a new sqlite file.""" + self._db_handle = dbapi2.connect(sqlite_file) + self._real_replica_uid = None + self._ensure_schema() + self._factory = document_factory or Document + + def set_document_factory(self, factory): + self._factory = factory + + def get_sync_target(self): + return SQLiteSyncTarget(self) + + @classmethod + def _which_index_storage(cls, c): + try: + c.execute("SELECT value FROM u1db_config" + " WHERE name = 'index_storage'") + except dbapi2.OperationalError, e: + # The table does not exist yet + return None, e + else: + return c.fetchone()[0], None + + WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.5 + + @classmethod + def _open_database(cls, sqlite_file, document_factory=None): + if not os.path.isfile(sqlite_file): + raise errors.DatabaseDoesNotExist() + tries = 2 + while True: + # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6) + # where without re-opening the database on Windows, it + # doesn't see the transaction that was just committed + db_handle = dbapi2.connect(sqlite_file) + c = db_handle.cursor() + v, err = cls._which_index_storage(c) + db_handle.close() + if v is not None: + break + # possibly another process is initializing it, wait for it to be + # done + if tries == 0: + raise err # go for the richest error? + tries -= 1 + time.sleep(cls.WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL) + return SQLiteDatabase._sqlite_registry[v]( + sqlite_file, document_factory=document_factory) + + @classmethod + def open_database(cls, sqlite_file, create, backend_cls=None, + document_factory=None): + try: + return cls._open_database( + sqlite_file, document_factory=document_factory) + except errors.DatabaseDoesNotExist: + if not create: + raise + if backend_cls is None: + # default is SQLitePartialExpandDatabase + backend_cls = SQLitePartialExpandDatabase + return backend_cls(sqlite_file, document_factory=document_factory) + + @staticmethod + def delete_database(sqlite_file): + try: + os.unlink(sqlite_file) + except OSError as ex: + if ex.errno == errno.ENOENT: + raise errors.DatabaseDoesNotExist() + raise + + @staticmethod + def register_implementation(klass): + """Register that we implement an SQLiteDatabase. + + The attribute _index_storage_value will be used as the lookup key. + """ + SQLiteDatabase._sqlite_registry[klass._index_storage_value] = klass + + def _get_sqlite_handle(self): + """Get access to the underlying sqlite database. + + This should only be used by the test suite, etc, for examining the + state of the underlying database. + """ + return self._db_handle + + def _close_sqlite_handle(self): + """Release access to the underlying sqlite database.""" + self._db_handle.close() + + def close(self): + self._close_sqlite_handle() + + def _is_initialized(self, c): + """Check if this database has been initialized.""" + c.execute("PRAGMA case_sensitive_like=ON") + try: + c.execute("SELECT value FROM u1db_config" + " WHERE name = 'sql_schema'") + except dbapi2.OperationalError: + # The table does not exist yet + val = None + else: + val = c.fetchone() + if val is not None: + return True + return False + + def _initialize(self, c): + """Create the schema in the database.""" + # read the script with sql commands + # TODO: Change how we set up the dependency. Most likely use something + # like lp:dirspec to grab the file from a common resource + # directory. Doesn't specifically need to be handled until we get + # to the point of packaging this. + schema_content = pkg_resources.resource_string( + __name__, 'dbschema.sql') + # Note: We'd like to use c.executescript() here, but it seems that + # executescript always commits, even if you set + # isolation_level = None, so if we want to properly handle + # exclusive locking and rollbacks between processes, we need + # to execute it line-by-line + for line in schema_content.split(';'): + if not line: + continue + c.execute(line) + # add extra fields + self._extra_schema_init(c) + # A unique identifier should be set for this replica. Implementations + # don't have to strictly use uuid here, but we do want the uid to be + # unique amongst all databases that will sync with each other. + # We might extend this to using something with hostname for easier + # debugging. + self._set_replica_uid_in_transaction(uuid.uuid4().hex) + c.execute("INSERT INTO u1db_config VALUES" " ('index_storage', ?)", + (self._index_storage_value,)) + + def _ensure_schema(self): + """Ensure that the database schema has been created.""" + old_isolation_level = self._db_handle.isolation_level + c = self._db_handle.cursor() + if self._is_initialized(c): + return + try: + # autocommit/own mgmt of transactions + self._db_handle.isolation_level = None + with self._db_handle: + # only one execution path should initialize the db + c.execute("begin exclusive") + if self._is_initialized(c): + return + self._initialize(c) + finally: + self._db_handle.isolation_level = old_isolation_level + + def _extra_schema_init(self, c): + """Add any extra fields, etc to the basic table definitions.""" + + def _parse_index_definition(self, index_field): + """Parse a field definition for an index, returning a Getter.""" + # Note: We may want to keep a Parser object around, and cache the + # Getter objects for a greater length of time. Specifically, if + # you create a bunch of indexes, and then insert 50k docs, you'll + # re-parse the indexes between puts. The time to insert the docs + # is still likely to dominate put_doc time, though. + parser = query_parser.Parser() + getter = parser.parse(index_field) + return getter + + def _update_indexes(self, doc_id, raw_doc, getters, db_cursor): + """Update document_fields for a single document. + + :param doc_id: Identifier for this document + :param raw_doc: The python dict representation of the document. + :param getters: A list of [(field_name, Getter)]. Getter.get will be + called to evaluate the index definition for this document, and the + results will be inserted into the db. + :param db_cursor: An sqlite Cursor. + :return: None + """ + values = [] + for field_name, getter in getters: + for idx_value in getter.get(raw_doc): + values.append((doc_id, field_name, idx_value)) + if values: + db_cursor.executemany( + "INSERT INTO document_fields VALUES (?, ?, ?)", values) + + def _set_replica_uid(self, replica_uid): + """Force the replica_uid to be set.""" + with self._db_handle: + self._set_replica_uid_in_transaction(replica_uid) + + def _set_replica_uid_in_transaction(self, replica_uid): + """Set the replica_uid. A transaction should already be held.""" + c = self._db_handle.cursor() + c.execute("INSERT OR REPLACE INTO u1db_config" + " VALUES ('replica_uid', ?)", + (replica_uid,)) + self._real_replica_uid = replica_uid + + def _get_replica_uid(self): + if self._real_replica_uid is not None: + return self._real_replica_uid + c = self._db_handle.cursor() + c.execute("SELECT value FROM u1db_config WHERE name = 'replica_uid'") + val = c.fetchone() + if val is None: + return None + self._real_replica_uid = val[0] + return self._real_replica_uid + + _replica_uid = property(_get_replica_uid) + + def _get_generation(self): + c = self._db_handle.cursor() + c.execute('SELECT max(generation) FROM transaction_log') + val = c.fetchone()[0] + if val is None: + return 0 + return val + + def _get_generation_info(self): + c = self._db_handle.cursor() + c.execute( + 'SELECT max(generation), transaction_id FROM transaction_log ') + val = c.fetchone() + if val[0] is None: + return(0, '') + return val + + def _get_trans_id_for_gen(self, generation): + if generation == 0: + return '' + c = self._db_handle.cursor() + c.execute( + 'SELECT transaction_id FROM transaction_log WHERE generation = ?', + (generation,)) + val = c.fetchone() + if val is None: + raise errors.InvalidGeneration + return val[0] + + def _get_transaction_log(self): + c = self._db_handle.cursor() + c.execute("SELECT doc_id, transaction_id FROM transaction_log" + " ORDER BY generation") + return c.fetchall() + + def _get_doc(self, doc_id, check_for_conflicts=False): + """Get just the document content, without fancy handling.""" + c = self._db_handle.cursor() + if check_for_conflicts: + c.execute( + "SELECT document.doc_rev, document.content, " + "count(conflicts.doc_rev) FROM document LEFT OUTER JOIN " + "conflicts ON conflicts.doc_id = document.doc_id WHERE " + "document.doc_id = ? GROUP BY document.doc_id, " + "document.doc_rev, document.content;", (doc_id,)) + else: + c.execute( + "SELECT doc_rev, content, 0 FROM document WHERE doc_id = ?", + (doc_id,)) + val = c.fetchone() + if val is None: + return None + doc_rev, content, conflicts = val + doc = self._factory(doc_id, doc_rev, content) + doc.has_conflicts = conflicts > 0 + return doc + + def _has_conflicts(self, doc_id): + c = self._db_handle.cursor() + c.execute("SELECT 1 FROM conflicts WHERE doc_id = ? LIMIT 1", + (doc_id,)) + val = c.fetchone() + if val is None: + return False + else: + return True + + def get_doc(self, doc_id, include_deleted=False): + doc = self._get_doc(doc_id, check_for_conflicts=True) + if doc is None: + return None + if doc.is_tombstone() and not include_deleted: + return None + return doc + + def get_all_docs(self, include_deleted=False): + """Get all documents from the database.""" + generation = self._get_generation() + results = [] + c = self._db_handle.cursor() + c.execute( + "SELECT document.doc_id, document.doc_rev, document.content, " + "count(conflicts.doc_rev) FROM document LEFT OUTER JOIN conflicts " + "ON conflicts.doc_id = document.doc_id GROUP BY document.doc_id, " + "document.doc_rev, document.content;") + rows = c.fetchall() + for doc_id, doc_rev, content, conflicts in rows: + if content is None and not include_deleted: + continue + doc = self._factory(doc_id, doc_rev, content) + doc.has_conflicts = conflicts > 0 + results.append(doc) + return (generation, results) + + def put_doc(self, doc): + if doc.doc_id is None: + raise errors.InvalidDocId() + self._check_doc_id(doc.doc_id) + self._check_doc_size(doc) + with self._db_handle: + old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) + if old_doc and old_doc.has_conflicts: + raise errors.ConflictedDoc() + if old_doc and doc.rev is None and old_doc.is_tombstone(): + new_rev = self._allocate_doc_rev(old_doc.rev) + else: + if old_doc is not None: + if old_doc.rev != doc.rev: + raise errors.RevisionConflict() + else: + if doc.rev is not None: + raise errors.RevisionConflict() + new_rev = self._allocate_doc_rev(doc.rev) + doc.rev = new_rev + self._put_and_update_indexes(old_doc, doc) + return new_rev + + def _expand_to_fields(self, doc_id, base_field, raw_doc, save_none): + """Convert a dict representation into named fields. + + So something like: {'key1': 'val1', 'key2': 'val2'} + gets converted into: [(doc_id, 'key1', 'val1', 0) + (doc_id, 'key2', 'val2', 0)] + :param doc_id: Just added to every record. + :param base_field: if set, these are nested keys, so each field should + be appropriately prefixed. + :param raw_doc: The python dictionary. + """ + # TODO: Handle lists + values = [] + for field_name, value in raw_doc.iteritems(): + if value is None and not save_none: + continue + if base_field: + full_name = base_field + '.' + field_name + else: + full_name = field_name + if value is None or isinstance(value, (int, float, basestring)): + values.append((doc_id, full_name, value, len(values))) + else: + subvalues = self._expand_to_fields(doc_id, full_name, value, + save_none) + for _, subfield_name, val, _ in subvalues: + values.append((doc_id, subfield_name, val, len(values))) + return values + + def _put_and_update_indexes(self, old_doc, doc): + """Actually insert a document into the database. + + This both updates the existing documents content, and any indexes that + refer to this document. + """ + raise NotImplementedError(self._put_and_update_indexes) + + def whats_changed(self, old_generation=0): + c = self._db_handle.cursor() + c.execute("SELECT generation, doc_id, transaction_id" + " FROM transaction_log" + " WHERE generation > ? ORDER BY generation DESC", + (old_generation,)) + results = c.fetchall() + cur_gen = old_generation + seen = set() + changes = [] + newest_trans_id = '' + for generation, doc_id, trans_id in results: + if doc_id not in seen: + changes.append((doc_id, generation, trans_id)) + seen.add(doc_id) + if changes: + cur_gen = changes[0][1] # max generation + newest_trans_id = changes[0][2] + changes.reverse() + else: + c.execute("SELECT generation, transaction_id" + " FROM transaction_log ORDER BY generation DESC LIMIT 1") + results = c.fetchone() + if not results: + cur_gen = 0 + newest_trans_id = '' + else: + cur_gen, newest_trans_id = results + + return cur_gen, newest_trans_id, changes + + def delete_doc(self, doc): + with self._db_handle: + old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) + if old_doc is None: + raise errors.DocumentDoesNotExist + if old_doc.rev != doc.rev: + raise errors.RevisionConflict() + if old_doc.is_tombstone(): + raise errors.DocumentAlreadyDeleted + if old_doc.has_conflicts: + raise errors.ConflictedDoc() + new_rev = self._allocate_doc_rev(doc.rev) + doc.rev = new_rev + doc.make_tombstone() + self._put_and_update_indexes(old_doc, doc) + return new_rev + + def _get_conflicts(self, doc_id): + c = self._db_handle.cursor() + c.execute("SELECT doc_rev, content FROM conflicts WHERE doc_id = ?", + (doc_id,)) + return [self._factory(doc_id, doc_rev, content) + for doc_rev, content in c.fetchall()] + + def get_doc_conflicts(self, doc_id): + with self._db_handle: + conflict_docs = self._get_conflicts(doc_id) + if not conflict_docs: + return [] + this_doc = self._get_doc(doc_id) + this_doc.has_conflicts = True + return [this_doc] + conflict_docs + + def _get_replica_gen_and_trans_id(self, other_replica_uid): + c = self._db_handle.cursor() + c.execute("SELECT known_generation, known_transaction_id FROM sync_log" + " WHERE replica_uid = ?", + (other_replica_uid,)) + val = c.fetchone() + if val is None: + other_gen = 0 + trans_id = '' + else: + other_gen = val[0] + trans_id = val[1] + return other_gen, trans_id + + def _set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, other_transaction_id): + with self._db_handle: + self._do_set_replica_gen_and_trans_id( + other_replica_uid, other_generation, other_transaction_id) + + def _do_set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, + other_transaction_id): + c = self._db_handle.cursor() + c.execute("INSERT OR REPLACE INTO sync_log VALUES (?, ?, ?)", + (other_replica_uid, other_generation, + other_transaction_id)) + + def _put_doc_if_newer(self, doc, save_conflict, replica_uid=None, + replica_gen=None, replica_trans_id=None): + with self._db_handle: + return super(SQLiteDatabase, self)._put_doc_if_newer( + doc, + save_conflict=save_conflict, + replica_uid=replica_uid, replica_gen=replica_gen, + replica_trans_id=replica_trans_id) + + def _add_conflict(self, c, doc_id, my_doc_rev, my_content): + c.execute("INSERT INTO conflicts VALUES (?, ?, ?)", + (doc_id, my_doc_rev, my_content)) + + def _delete_conflicts(self, c, doc, conflict_revs): + deleting = [(doc.doc_id, c_rev) for c_rev in conflict_revs] + c.executemany("DELETE FROM conflicts" + " WHERE doc_id=? AND doc_rev=?", deleting) + doc.has_conflicts = self._has_conflicts(doc.doc_id) + + def _prune_conflicts(self, doc, doc_vcr): + if self._has_conflicts(doc.doc_id): + autoresolved = False + c_revs_to_prune = [] + for c_doc in self._get_conflicts(doc.doc_id): + c_vcr = vectorclock.VectorClockRev(c_doc.rev) + if doc_vcr.is_newer(c_vcr): + c_revs_to_prune.append(c_doc.rev) + elif doc.same_content_as(c_doc): + c_revs_to_prune.append(c_doc.rev) + doc_vcr.maximize(c_vcr) + autoresolved = True + if autoresolved: + doc_vcr.increment(self._replica_uid) + doc.rev = doc_vcr.as_str() + c = self._db_handle.cursor() + self._delete_conflicts(c, doc, c_revs_to_prune) + + def _force_doc_sync_conflict(self, doc): + my_doc = self._get_doc(doc.doc_id) + c = self._db_handle.cursor() + self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev)) + self._add_conflict(c, doc.doc_id, my_doc.rev, my_doc.get_json()) + doc.has_conflicts = True + self._put_and_update_indexes(my_doc, doc) + + def resolve_doc(self, doc, conflicted_doc_revs): + with self._db_handle: + cur_doc = self._get_doc(doc.doc_id) + # TODO: https://bugs.launchpad.net/u1db/+bug/928274 + # I think we have a logic bug in resolve_doc + # Specifically, cur_doc.rev is always in the final vector + # clock of revisions that we supersede, even if it wasn't in + # conflicted_doc_revs. We still add it as a conflict, but the + # fact that _put_doc_if_newer propagates resolutions means I + # think that conflict could accidentally be resolved. We need + # to add a test for this case first. (create a rev, create a + # conflict, create another conflict, resolve the first rev + # and first conflict, then make sure that the resolved + # rev doesn't supersede the second conflict rev.) It *might* + # not matter, because the superseding rev is in as a + # conflict, but it does seem incorrect + new_rev = self._ensure_maximal_rev(cur_doc.rev, + conflicted_doc_revs) + superseded_revs = set(conflicted_doc_revs) + c = self._db_handle.cursor() + doc.rev = new_rev + if cur_doc.rev in superseded_revs: + self._put_and_update_indexes(cur_doc, doc) + else: + self._add_conflict(c, doc.doc_id, new_rev, doc.get_json()) + # TODO: Is there some way that we could construct a rev that would + # end up in superseded_revs, such that we add a conflict, and + # then immediately delete it? + self._delete_conflicts(c, doc, superseded_revs) + + def list_indexes(self): + """Return the list of indexes and their definitions.""" + c = self._db_handle.cursor() + # TODO: How do we test the ordering? + c.execute("SELECT name, field FROM index_definitions" + " ORDER BY name, offset") + definitions = [] + cur_name = None + for name, field in c.fetchall(): + if cur_name != name: + definitions.append((name, [])) + cur_name = name + definitions[-1][-1].append(field) + return definitions + + def _get_index_definition(self, index_name): + """Return the stored definition for a given index_name.""" + c = self._db_handle.cursor() + c.execute("SELECT field FROM index_definitions" + " WHERE name = ? ORDER BY offset", (index_name,)) + fields = [x[0] for x in c.fetchall()] + if not fields: + raise errors.IndexDoesNotExist + return fields + + @staticmethod + def _strip_glob(value): + """Remove the trailing * from a value.""" + assert value[-1] == '*' + return value[:-1] + + def _format_query(self, definition, key_values): + # First, build the definition. We join the document_fields table + # against itself, as many times as the 'width' of our definition. + # We then do a query for each key_value, one-at-a-time. + # Note: All of these strings are static, we could cache them, etc. + tables = ["document_fields d%d" % i for i in range(len(definition))] + novalue_where = ["d.doc_id = d%d.doc_id" + " AND d%d.field_name = ?" + % (i, i) for i in range(len(definition))] + wildcard_where = [novalue_where[i] + + (" AND d%d.value NOT NULL" % (i,)) + for i in range(len(definition))] + exact_where = [novalue_where[i] + + (" AND d%d.value = ?" % (i,)) + for i in range(len(definition))] + like_where = [novalue_where[i] + + (" AND d%d.value GLOB ?" % (i,)) + for i in range(len(definition))] + is_wildcard = False + # Merge the lists together, so that: + # [field1, field2, field3], [val1, val2, val3] + # Becomes: + # (field1, val1, field2, val2, field3, val3) + args = [] + where = [] + for idx, (field, value) in enumerate(zip(definition, key_values)): + args.append(field) + if value.endswith('*'): + if value == '*': + where.append(wildcard_where[idx]) + else: + # This is a glob match + if is_wildcard: + # We can't have a partial wildcard following + # another wildcard + raise errors.InvalidGlobbing + where.append(like_where[idx]) + args.append(value) + is_wildcard = True + else: + if is_wildcard: + raise errors.InvalidGlobbing + where.append(exact_where[idx]) + args.append(value) + statement = ( + "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " + "document d, %s LEFT OUTER JOIN conflicts c ON c.doc_id = " + "d.doc_id WHERE %s GROUP BY d.doc_id, d.doc_rev, d.content ORDER " + "BY %s;" % (', '.join(tables), ' AND '.join(where), ', '.join( + ['d%d.value' % i for i in range(len(definition))]))) + return statement, args + + def get_from_index(self, index_name, *key_values): + definition = self._get_index_definition(index_name) + if len(key_values) != len(definition): + raise errors.InvalidValueForIndex() + statement, args = self._format_query(definition, key_values) + c = self._db_handle.cursor() + try: + c.execute(statement, tuple(args)) + except dbapi2.OperationalError, e: + raise dbapi2.OperationalError( + str(e) + + '\nstatement: %s\nargs: %s\n' % (statement, args)) + res = c.fetchall() + results = [] + for row in res: + doc = self._factory(row[0], row[1], row[2]) + doc.has_conflicts = row[3] > 0 + results.append(doc) + return results + + def _format_range_query(self, definition, start_value, end_value): + tables = ["document_fields d%d" % i for i in range(len(definition))] + novalue_where = [ + "d.doc_id = d%d.doc_id AND d%d.field_name = ?" % (i, i) for i in + range(len(definition))] + wildcard_where = [ + novalue_where[i] + (" AND d%d.value NOT NULL" % (i,)) for i in + range(len(definition))] + like_where = [ + novalue_where[i] + ( + " AND (d%d.value < ? OR d%d.value GLOB ?)" % (i, i)) for i in + range(len(definition))] + range_where_lower = [ + novalue_where[i] + (" AND d%d.value >= ?" % (i,)) for i in + range(len(definition))] + range_where_upper = [ + novalue_where[i] + (" AND d%d.value <= ?" % (i,)) for i in + range(len(definition))] + args = [] + where = [] + if start_value: + if isinstance(start_value, basestring): + start_value = (start_value,) + if len(start_value) != len(definition): + raise errors.InvalidValueForIndex() + is_wildcard = False + for idx, (field, value) in enumerate(zip(definition, start_value)): + args.append(field) + if value.endswith('*'): + if value == '*': + where.append(wildcard_where[idx]) + else: + # This is a glob match + if is_wildcard: + # We can't have a partial wildcard following + # another wildcard + raise errors.InvalidGlobbing + where.append(range_where_lower[idx]) + args.append(self._strip_glob(value)) + is_wildcard = True + else: + if is_wildcard: + raise errors.InvalidGlobbing + where.append(range_where_lower[idx]) + args.append(value) + if end_value: + if isinstance(end_value, basestring): + end_value = (end_value,) + if len(end_value) != len(definition): + raise errors.InvalidValueForIndex() + is_wildcard = False + for idx, (field, value) in enumerate(zip(definition, end_value)): + args.append(field) + if value.endswith('*'): + if value == '*': + where.append(wildcard_where[idx]) + else: + # This is a glob match + if is_wildcard: + # We can't have a partial wildcard following + # another wildcard + raise errors.InvalidGlobbing + where.append(like_where[idx]) + args.append(self._strip_glob(value)) + args.append(value) + is_wildcard = True + else: + if is_wildcard: + raise errors.InvalidGlobbing + where.append(range_where_upper[idx]) + args.append(value) + statement = ( + "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " + "document d, %s LEFT OUTER JOIN conflicts c ON c.doc_id = " + "d.doc_id WHERE %s GROUP BY d.doc_id, d.doc_rev, d.content ORDER " + "BY %s;" % (', '.join(tables), ' AND '.join(where), ', '.join( + ['d%d.value' % i for i in range(len(definition))]))) + return statement, args + + def get_range_from_index(self, index_name, start_value=None, + end_value=None): + """Return all documents with key values in the specified range.""" + definition = self._get_index_definition(index_name) + statement, args = self._format_range_query( + definition, start_value, end_value) + c = self._db_handle.cursor() + try: + c.execute(statement, tuple(args)) + except dbapi2.OperationalError, e: + raise dbapi2.OperationalError( + str(e) + + '\nstatement: %s\nargs: %s\n' % (statement, args)) + res = c.fetchall() + results = [] + for row in res: + doc = self._factory(row[0], row[1], row[2]) + doc.has_conflicts = row[3] > 0 + results.append(doc) + return results + + def get_index_keys(self, index_name): + c = self._db_handle.cursor() + definition = self._get_index_definition(index_name) + value_fields = ', '.join([ + 'd%d.value' % i for i in range(len(definition))]) + tables = ["document_fields d%d" % i for i in range(len(definition))] + novalue_where = [ + "d.doc_id = d%d.doc_id AND d%d.field_name = ?" % (i, i) for i in + range(len(definition))] + where = [ + novalue_where[i] + (" AND d%d.value NOT NULL" % (i,)) for i in + range(len(definition))] + statement = ( + "SELECT %s FROM document d, %s WHERE %s GROUP BY %s;" % ( + value_fields, ', '.join(tables), ' AND '.join(where), + value_fields)) + try: + c.execute(statement, tuple(definition)) + except dbapi2.OperationalError, e: + raise dbapi2.OperationalError( + str(e) + + '\nstatement: %s\nargs: %s\n' % (statement, tuple(definition))) + return c.fetchall() + + def delete_index(self, index_name): + with self._db_handle: + c = self._db_handle.cursor() + c.execute("DELETE FROM index_definitions WHERE name = ?", + (index_name,)) + c.execute( + "DELETE FROM document_fields WHERE document_fields.field_name " + " NOT IN (SELECT field from index_definitions)") + + +class SQLiteSyncTarget(CommonSyncTarget): + + def get_sync_info(self, source_replica_uid): + source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id( + source_replica_uid) + my_gen, my_trans_id = self._db._get_generation_info() + return ( + self._db._replica_uid, my_gen, my_trans_id, source_gen, + source_trans_id) + + def record_sync_info(self, source_replica_uid, source_replica_generation, + source_replica_transaction_id): + if self._trace_hook: + self._trace_hook('record_sync_info') + self._db._set_replica_gen_and_trans_id( + source_replica_uid, source_replica_generation, + source_replica_transaction_id) + + +class SQLitePartialExpandDatabase(SQLiteDatabase): + """An SQLite Backend that expands documents into a document_field table. + + It stores the original document text in document.doc. For fields that are + indexed, the data goes into document_fields. + """ + + _index_storage_value = 'expand referenced' + + def _get_indexed_fields(self): + """Determine what fields are indexed.""" + c = self._db_handle.cursor() + c.execute("SELECT field FROM index_definitions") + return set([x[0] for x in c.fetchall()]) + + def _evaluate_index(self, raw_doc, field): + parser = query_parser.Parser() + getter = parser.parse(field) + return getter.get(raw_doc) + + def _put_and_update_indexes(self, old_doc, doc): + c = self._db_handle.cursor() + if doc and not doc.is_tombstone(): + raw_doc = json.loads(doc.get_json()) + else: + raw_doc = {} + if old_doc is not None: + c.execute("UPDATE document SET doc_rev=?, content=?" + " WHERE doc_id = ?", + (doc.rev, doc.get_json(), doc.doc_id)) + c.execute("DELETE FROM document_fields WHERE doc_id = ?", + (doc.doc_id,)) + else: + c.execute("INSERT INTO document (doc_id, doc_rev, content)" + " VALUES (?, ?, ?)", + (doc.doc_id, doc.rev, doc.get_json())) + indexed_fields = self._get_indexed_fields() + if indexed_fields: + # It is expected that len(indexed_fields) is shorter than + # len(raw_doc) + getters = [(field, self._parse_index_definition(field)) + for field in indexed_fields] + self._update_indexes(doc.doc_id, raw_doc, getters, c) + trans_id = self._allocate_transaction_id() + c.execute("INSERT INTO transaction_log(doc_id, transaction_id)" + " VALUES (?, ?)", (doc.doc_id, trans_id)) + + def create_index(self, index_name, *index_expressions): + with self._db_handle: + c = self._db_handle.cursor() + cur_fields = self._get_indexed_fields() + definition = [(index_name, idx, field) + for idx, field in enumerate(index_expressions)] + try: + c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", + definition) + except dbapi2.IntegrityError as e: + stored_def = self._get_index_definition(index_name) + if stored_def == [x[-1] for x in definition]: + return + raise errors.IndexNameTakenError, e, sys.exc_info()[2] + new_fields = set( + [f for f in index_expressions if f not in cur_fields]) + if new_fields: + self._update_all_indexes(new_fields) + + def _iter_all_docs(self): + c = self._db_handle.cursor() + c.execute("SELECT doc_id, content FROM document") + while True: + next_rows = c.fetchmany() + if not next_rows: + break + for row in next_rows: + yield row + + def _update_all_indexes(self, new_fields): + """Iterate all the documents, and add content to document_fields. + + :param new_fields: The index definitions that need to be added. + """ + getters = [(field, self._parse_index_definition(field)) + for field in new_fields] + c = self._db_handle.cursor() + for doc_id, doc in self._iter_all_docs(): + if doc is None: + continue + raw_doc = json.loads(doc) + self._update_indexes(doc_id, raw_doc, getters, c) + +SQLiteDatabase.register_implementation(SQLitePartialExpandDatabase) diff --git a/common/src/leap/soledad/common/l2db/errors.py b/common/src/leap/soledad/common/l2db/errors.py new file mode 100644 index 00000000..b502fc2d --- /dev/null +++ b/common/src/leap/soledad/common/l2db/errors.py @@ -0,0 +1,194 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""A list of errors that u1db can raise.""" + + +class U1DBError(Exception): + """Generic base class for U1DB errors.""" + + # description/tag for identifying the error during transmission (http,...) + wire_description = "error" + + def __init__(self, message=None): + self.message = message + + +class RevisionConflict(U1DBError): + """The document revisions supplied does not match the current version.""" + + wire_description = "revision conflict" + + +class InvalidJSON(U1DBError): + """Content was not valid json.""" + + +class InvalidContent(U1DBError): + """Content was not a python dictionary.""" + + +class InvalidDocId(U1DBError): + """A document was requested with an invalid document identifier.""" + + wire_description = "invalid document id" + + +class MissingDocIds(U1DBError): + """Needs document ids.""" + + wire_description = "missing document ids" + + +class DocumentTooBig(U1DBError): + """Document exceeds the maximum document size for this database.""" + + wire_description = "document too big" + + +class UserQuotaExceeded(U1DBError): + """Document exceeds the maximum document size for this database.""" + + wire_description = "user quota exceeded" + + +class SubscriptionNeeded(U1DBError): + """User needs a subscription to be able to use this replica..""" + + wire_description = "user needs subscription" + + +class InvalidTransactionId(U1DBError): + """Invalid transaction for generation.""" + + wire_description = "invalid transaction id" + + +class InvalidGeneration(U1DBError): + """Generation was previously synced with a different transaction id.""" + + wire_description = "invalid generation" + + +class InvalidReplicaUID(U1DBError): + """Attempting to sync a database with itself.""" + + wire_description = "invalid replica uid" + + +class ConflictedDoc(U1DBError): + """The document is conflicted, you must call resolve before put()""" + + +class InvalidValueForIndex(U1DBError): + """The values supplied does not match the index definition.""" + + +class InvalidGlobbing(U1DBError): + """Raised if wildcard matches are not strictly at the tail of the request. + """ + + +class DocumentDoesNotExist(U1DBError): + """The document does not exist.""" + + wire_description = "document does not exist" + + +class DocumentAlreadyDeleted(U1DBError): + """The document was already deleted.""" + + wire_description = "document already deleted" + + +class DatabaseDoesNotExist(U1DBError): + """The database does not exist.""" + + wire_description = "database does not exist" + + +class IndexNameTakenError(U1DBError): + """The given index name is already taken.""" + + +class IndexDefinitionParseError(U1DBError): + """The index definition cannot be parsed.""" + + +class IndexDoesNotExist(U1DBError): + """No index of that name exists.""" + + +class Unauthorized(U1DBError): + """Request wasn't authorized properly.""" + + wire_description = "unauthorized" + + +class HTTPError(U1DBError): + """Unspecific HTTP errror.""" + + wire_description = None + + def __init__(self, status, message=None, headers={}): + self.status = status + self.message = message + self.headers = headers + + def __str__(self): + if not self.message: + return "HTTPError(%d)" % self.status + else: + return "HTTPError(%d, %r)" % (self.status, self.message) + + +class Unavailable(HTTPError): + """Server not available not serve request.""" + + wire_description = "unavailable" + + def __init__(self, message=None, headers={}): + super(Unavailable, self).__init__(503, message, headers) + + def __str__(self): + if not self.message: + return "Unavailable()" + else: + return "Unavailable(%r)" % self.message + + +class BrokenSyncStream(U1DBError): + """Unterminated or otherwise broken sync exchange stream.""" + + wire_description = None + + +class UnknownAuthMethod(U1DBError): + """Unknown auhorization method.""" + + wire_description = None + + +# mapping wire (transimission) descriptions/tags for errors to the exceptions +wire_description_to_exc = dict( + (x.wire_description, x) for x in globals().values() + if getattr(x, 'wire_description', None) not in (None, "error")) +wire_description_to_exc["error"] = U1DBError + + +# +# wire error descriptions not corresponding to an exception +DOCUMENT_DELETED = "document deleted" diff --git a/common/src/leap/soledad/common/l2db/query_parser.py b/common/src/leap/soledad/common/l2db/query_parser.py new file mode 100644 index 00000000..dd35b12a --- /dev/null +++ b/common/src/leap/soledad/common/l2db/query_parser.py @@ -0,0 +1,371 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. +""" +Code for parsing Index definitions. +""" + +import re + +from leap.soledad.common.l2db import errors + + +class Getter(object): + """Get values from a document based on a specification.""" + + def get(self, raw_doc): + """Get a value from the document. + + :param raw_doc: a python dictionary to get the value from. + :return: A list of values that match the description. + """ + raise NotImplementedError(self.get) + + +class StaticGetter(Getter): + """A getter that returns a defined value (independent of the doc).""" + + def __init__(self, value): + """Create a StaticGetter. + + :param value: the value to return when get is called. + """ + if value is None: + self.value = [] + elif isinstance(value, list): + self.value = value + else: + self.value = [value] + + def get(self, raw_doc): + return self.value + + +def extract_field(raw_doc, subfields, index=0): + if not isinstance(raw_doc, dict): + return [] + val = raw_doc.get(subfields[index]) + if val is None: + return [] + if index < len(subfields) - 1: + if isinstance(val, list): + results = [] + for item in val: + results.extend(extract_field(item, subfields, index + 1)) + return results + if isinstance(val, dict): + return extract_field(val, subfields, index + 1) + return [] + if isinstance(val, dict): + return [] + if isinstance(val, list): + # Strip anything in the list that isn't a simple type + return [v for v in val if not isinstance(v, (dict, list))] + return [val] + + +class ExtractField(Getter): + """Extract a field from the document.""" + + def __init__(self, field): + """Create an ExtractField object. + + When a document is passed to get() this will return a value + from the document based on the field specifier passed to + the constructor. + + None will be returned if the field is nonexistant, or refers to an + object, rather than a simple type or list of simple types. + + :param field: a specifier for the field to return. + This is either a field name, or a dotted field name. + """ + self.field = field.split('.') + + def get(self, raw_doc): + return extract_field(raw_doc, self.field) + + +class Transformation(Getter): + """A transformation on a value from another Getter.""" + + name = None + arity = 1 + args = ['expression'] + + def __init__(self, inner): + """Create a transformation. + + :param inner: the argument(s) to the transformation. + """ + self.inner = inner + + def get(self, raw_doc): + inner_values = self.inner.get(raw_doc) + assert isinstance(inner_values, list),\ + 'get() should always return a list' + return self.transform(inner_values) + + def transform(self, values): + """Transform the values. + + This should be implemented by subclasses to transform the + value when get() is called. + + :param values: the values from the other Getter + :return: the transformed values. + """ + raise NotImplementedError(self.transform) + + +class Lower(Transformation): + """Lowercase a string. + + This transformation will return None for non-string inputs. However, + it will lowercase any strings in a list, dropping any elements + that are not strings. + """ + + name = "lower" + + def _can_transform(self, val): + return isinstance(val, basestring) + + def transform(self, values): + if not values: + return [] + return [val.lower() for val in values if self._can_transform(val)] + + +class Number(Transformation): + """Convert an integer to a zero padded string. + + This transformation will return None for non-integer inputs. However, it + will transform any integers in a list, dropping any elements that are not + integers. + """ + + name = 'number' + arity = 2 + args = ['expression', int] + + def __init__(self, inner, number): + super(Number, self).__init__(inner) + self.padding = "%%0%sd" % number + + def _can_transform(self, val): + return isinstance(val, int) and not isinstance(val, bool) + + def transform(self, values): + """Transform any integers in values into zero padded strings.""" + if not values: + return [] + return [self.padding % (v,) for v in values if self._can_transform(v)] + + +class Bool(Transformation): + """Convert bool to string.""" + + name = "bool" + args = ['expression'] + + def _can_transform(self, val): + return isinstance(val, bool) + + def transform(self, values): + """Transform any booleans in values into strings.""" + if not values: + return [] + return [('1' if v else '0') for v in values if self._can_transform(v)] + + +class SplitWords(Transformation): + """Split a string on whitespace. + + This Getter will return [] for non-string inputs. It will however + split any strings in an input list, discarding any elements that + are not strings. + """ + + name = "split_words" + + def _can_transform(self, val): + return isinstance(val, basestring) + + def transform(self, values): + if not values: + return [] + result = set() + for value in values: + if self._can_transform(value): + for word in value.split(): + result.add(word) + return list(result) + + +class Combine(Transformation): + """Combine multiple expressions into a single index.""" + + name = "combine" + # variable number of args + arity = -1 + + def __init__(self, *inner): + super(Combine, self).__init__(inner) + + def get(self, raw_doc): + inner_values = [] + for inner in self.inner: + inner_values.extend(inner.get(raw_doc)) + return self.transform(inner_values) + + def transform(self, values): + return values + + +class IsNull(Transformation): + """Indicate whether the input is None. + + This Getter returns a bool indicating whether the input is nil. + """ + + name = "is_null" + + def transform(self, values): + return [len(values) == 0] + + +def check_fieldname(fieldname): + if fieldname.endswith('.'): + raise errors.IndexDefinitionParseError( + "Fieldname cannot end in '.':%s^" % (fieldname,)) + + +class Parser(object): + """Parse an index expression into a sequence of transformations.""" + + _transformations = {} + _delimiters = re.compile("\(|\)|,") + + def __init__(self): + self._tokens = [] + + def _set_expression(self, expression): + self._open_parens = 0 + self._tokens = [] + expression = expression.strip() + while expression: + delimiter = self._delimiters.search(expression) + if delimiter: + idx = delimiter.start() + if idx == 0: + result, expression = (expression[:1], expression[1:]) + self._tokens.append(result) + else: + result, expression = (expression[:idx], expression[idx:]) + result = result.strip() + if result: + self._tokens.append(result) + else: + expression = expression.strip() + if expression: + self._tokens.append(expression) + expression = None + + def _get_token(self): + if self._tokens: + return self._tokens.pop(0) + + def _peek_token(self): + if self._tokens: + return self._tokens[0] + + @staticmethod + def _to_getter(term): + if isinstance(term, Getter): + return term + check_fieldname(term) + return ExtractField(term) + + def _parse_op(self, op_name): + self._get_token() # '(' + op = self._transformations.get(op_name, None) + if op is None: + raise errors.IndexDefinitionParseError( + "Unknown operation: %s" % op_name) + args = [] + while True: + args.append(self._parse_term()) + sep = self._get_token() + if sep == ')': + break + if sep != ',': + raise errors.IndexDefinitionParseError( + "Unexpected token '%s' in parentheses." % (sep,)) + parsed = [] + for i, arg in enumerate(args): + arg_type = op.args[i % len(op.args)] + if arg_type == 'expression': + inner = self._to_getter(arg) + else: + try: + inner = arg_type(arg) + except ValueError, e: + raise errors.IndexDefinitionParseError( + "Invalid value %r for argument type %r " + "(%r)." % (arg, arg_type, e)) + parsed.append(inner) + return op(*parsed) + + def _parse_term(self): + term = self._get_token() + if term is None: + raise errors.IndexDefinitionParseError( + "Unexpected end of index definition.") + if term in (',', ')', '('): + raise errors.IndexDefinitionParseError( + "Unexpected token '%s' at start of expression." % (term,)) + next_token = self._peek_token() + if next_token == '(': + return self._parse_op(term) + return term + + def parse(self, expression): + self._set_expression(expression) + term = self._to_getter(self._parse_term()) + if self._peek_token(): + raise errors.IndexDefinitionParseError( + "Unexpected token '%s' after end of expression." + % (self._peek_token(),)) + return term + + def parse_all(self, fields): + return [self.parse(field) for field in fields] + + @classmethod + def register_transormation(cls, transform): + assert transform.name not in cls._transformations, ( + "Transform %s already registered for %s" + % (transform.name, cls._transformations[transform.name])) + cls._transformations[transform.name] = transform + + +Parser.register_transormation(SplitWords) +Parser.register_transormation(Lower) +Parser.register_transormation(Number) +Parser.register_transormation(Bool) +Parser.register_transormation(IsNull) +Parser.register_transormation(Combine) diff --git a/common/src/leap/soledad/common/l2db/remote/__init__.py b/common/src/leap/soledad/common/l2db/remote/__init__.py new file mode 100644 index 00000000..3f32e381 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. diff --git a/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py b/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py new file mode 100644 index 00000000..a2cbff62 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py @@ -0,0 +1,68 @@ +# Copyright 2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. +"""U1DB Basic Auth authorisation WSGI middleware.""" +import httplib +try: + import simplejson as json +except ImportError: + import json # noqa +from wsgiref.util import shift_path_info + + +class Unauthorized(Exception): + """User authorization failed.""" + + +class BasicAuthMiddleware(object): + """U1DB Basic Auth Authorisation WSGI middleware.""" + + def __init__(self, app, prefix): + self.app = app + self.prefix = prefix + + def _error(self, start_response, status, description, message=None): + start_response("%d %s" % (status, httplib.responses[status]), + [('content-type', 'application/json')]) + err = {"error": description} + if message: + err['message'] = message + return [json.dumps(err)] + + def __call__(self, environ, start_response): + if self.prefix and not environ['PATH_INFO'].startswith(self.prefix): + return self._error(start_response, 400, "bad request") + auth = environ.get('HTTP_AUTHORIZATION') + if not auth: + return self._error(start_response, 401, "unauthorized", + "Missing Basic Authentication.") + scheme, encoded = auth.split(None, 1) + if scheme.lower() != 'basic': + return self._error( + start_response, 401, "unauthorized", + "Missing Basic Authentication") + user, password = encoded.decode('base64').split(':', 1) + try: + self.verify_user(environ, user, password) + except Unauthorized: + return self._error( + start_response, 401, "unauthorized", + "Incorrect password or login.") + del environ['HTTP_AUTHORIZATION'] + shift_path_info(environ) + return self.app(environ, start_response) + + def verify_user(self, environ, username, password): + raise NotImplementedError(self.verify_user) diff --git a/common/src/leap/soledad/common/l2db/remote/http_app.py b/common/src/leap/soledad/common/l2db/remote/http_app.py new file mode 100644 index 00000000..65277bd1 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_app.py @@ -0,0 +1,660 @@ +# Copyright 2011-2012 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +""" +HTTP Application exposing U1DB. +""" +# TODO -- deprecate, use twisted/txaio. + +import functools +import httplib +import inspect +try: + import simplejson as json +except ImportError: + import json # noqa +import sys +import urlparse + +import routes.mapper + +from leap.soledad.common.l2db import ( + __version__ as _u1db_version, + DBNAME_CONSTRAINTS, Document, + errors, sync) +from leap.soledad.common.l2db.remote import http_errors, utils + + +def parse_bool(expression): + """Parse boolean querystring parameter.""" + if expression == 'true': + return True + return False + + +def parse_list(expression): + if not expression: + return [] + return [t.strip() for t in expression.split(',')] + + +def none_or_str(expression): + if expression is None: + return None + return str(expression) + + +class BadRequest(Exception): + """Bad request.""" + + +class _FencedReader(object): + """Read and get lines from a file but not past a given length.""" + + MAXCHUNK = 8192 + + def __init__(self, rfile, total, max_entry_size): + self.rfile = rfile + self.remaining = total + self.max_entry_size = max_entry_size + self._kept = None + + def read_chunk(self, atmost): + if self._kept is not None: + # ignore atmost, kept data should be a subchunk anyway + kept, self._kept = self._kept, None + return kept + if self.remaining == 0: + return '' + data = self.rfile.read(min(self.remaining, atmost)) + self.remaining -= len(data) + return data + + def getline(self): + line_parts = [] + size = 0 + while True: + chunk = self.read_chunk(self.MAXCHUNK) + if chunk == '': + break + nl = chunk.find("\n") + if nl != -1: + size += nl + 1 + if size > self.max_entry_size: + raise BadRequest + line_parts.append(chunk[:nl + 1]) + rest = chunk[nl + 1:] + self._kept = rest or None + break + else: + size += len(chunk) + if size > self.max_entry_size: + raise BadRequest + line_parts.append(chunk) + return ''.join(line_parts) + + +def http_method(**control): + """Decoration for handling of query arguments and content for a HTTP + method. + + args and content here are the query arguments and body of the incoming + HTTP requests. + + Match query arguments to python method arguments: + w = http_method()(f) + w(self, args, content) => args["content"]=content; + f(self, **args) + + JSON deserialize content to arguments: + w = http_method(content_as_args=True,...)(f) + w(self, args, content) => args.update(json.loads(content)); + f(self, **args) + + Support conversions (e.g int): + w = http_method(Arg=Conv,...)(f) + w(self, args, content) => args["Arg"]=Conv(args["Arg"]); + f(self, **args) + + Enforce no use of query arguments: + w = http_method(no_query=True,...)(f) + w(self, args, content) raises BadRequest if args is not empty + + Argument mismatches, deserialisation failures produce BadRequest. + """ + content_as_args = control.pop('content_as_args', False) + no_query = control.pop('no_query', False) + conversions = control.items() + + def wrap(f): + argspec = inspect.getargspec(f) + assert argspec.args[0] == "self" + nargs = len(argspec.args) + ndefaults = len(argspec.defaults or ()) + required_args = set(argspec.args[1:nargs - ndefaults]) + all_args = set(argspec.args) + + @functools.wraps(f) + def wrapper(self, args, content): + if no_query and args: + raise BadRequest() + if content is not None: + if content_as_args: + try: + args.update(json.loads(content)) + except ValueError: + raise BadRequest() + else: + args["content"] = content + if not (required_args <= set(args) <= all_args): + raise BadRequest("Missing required arguments.") + for name, conv in conversions: + if name not in args: + continue + try: + args[name] = conv(args[name]) + except ValueError: + raise BadRequest() + return f(self, **args) + + return wrapper + + return wrap + + +class URLToResource(object): + """Mappings from URLs to resources.""" + + def __init__(self): + self._map = routes.mapper.Mapper(controller_scan=None) + + def register(self, resource_cls): + # register + self._map.connect(None, resource_cls.url_pattern, + resource_cls=resource_cls, + requirements={"dbname": DBNAME_CONSTRAINTS}) + self._map.create_regs() + return resource_cls + + def match(self, path): + params = self._map.match(path) + if params is None: + return None, None + resource_cls = params.pop('resource_cls') + return resource_cls, params + +url_to_resource = URLToResource() + + +@url_to_resource.register +class GlobalResource(object): + """Global (root) resource.""" + + url_pattern = "/" + + def __init__(self, state, responder): + self.state = state + self.responder = responder + + @http_method() + def get(self): + info = self.state.global_info() + info['version'] = _u1db_version + self.responder.send_response_json(**info) + + +@url_to_resource.register +class DatabaseResource(object): + """Database resource.""" + + url_pattern = "/{dbname}" + + def __init__(self, dbname, state, responder): + self.dbname = dbname + self.state = state + self.responder = responder + + @http_method() + def get(self): + self.state.check_database(self.dbname) + self.responder.send_response_json(200) + + @http_method(content_as_args=True) + def put(self): + self.state.ensure_database(self.dbname) + self.responder.send_response_json(200, ok=True) + + @http_method() + def delete(self): + self.state.delete_database(self.dbname) + self.responder.send_response_json(200, ok=True) + + +@url_to_resource.register +class DocsResource(object): + """Documents resource.""" + + url_pattern = "/{dbname}/docs" + + def __init__(self, dbname, state, responder): + self.responder = responder + self.db = state.open_database(dbname) + + @http_method(doc_ids=parse_list, check_for_conflicts=parse_bool, + include_deleted=parse_bool) + def get(self, doc_ids=None, check_for_conflicts=True, + include_deleted=False): + if doc_ids is None: + raise errors.MissingDocIds + docs = self.db.get_docs(doc_ids, include_deleted=include_deleted) + self.responder.content_type = 'application/json' + self.responder.start_response(200) + self.responder.start_stream(), + for doc in docs: + entry = dict( + doc_id=doc.doc_id, doc_rev=doc.rev, content=doc.get_json(), + has_conflicts=doc.has_conflicts) + self.responder.stream_entry(entry) + self.responder.end_stream() + self.responder.finish_response() + + +@url_to_resource.register +class AllDocsResource(object): + """All Documents resource.""" + + url_pattern = "/{dbname}/all-docs" + + def __init__(self, dbname, state, responder): + self.responder = responder + self.db = state.open_database(dbname) + + @http_method(include_deleted=parse_bool) + def get(self, include_deleted=False): + gen, docs = self.db.get_all_docs(include_deleted=include_deleted) + self.responder.content_type = 'application/json' + # returning a x-u1db-generation header is optional + # HTTPDatabase will fallback to return -1 if it's missing + self.responder.start_response(200, + headers={'x-u1db-generation': str(gen)}) + self.responder.start_stream(), + for doc in docs: + entry = dict( + doc_id=doc.doc_id, doc_rev=doc.rev, content=doc.get_json(), + has_conflicts=doc.has_conflicts) + self.responder.stream_entry(entry) + self.responder.end_stream() + self.responder.finish_response() + + +@url_to_resource.register +class DocResource(object): + """Document resource.""" + + url_pattern = "/{dbname}/doc/{id:.*}" + + def __init__(self, dbname, id, state, responder): + self.id = id + self.responder = responder + self.db = state.open_database(dbname) + + @http_method(old_rev=str) + def put(self, content, old_rev=None): + doc = Document(self.id, old_rev, content) + doc_rev = self.db.put_doc(doc) + if old_rev is None: + status = 201 # created + else: + status = 200 + self.responder.send_response_json(status, rev=doc_rev) + + @http_method(old_rev=str) + def delete(self, old_rev=None): + doc = Document(self.id, old_rev, None) + self.db.delete_doc(doc) + self.responder.send_response_json(200, rev=doc.rev) + + @http_method(include_deleted=parse_bool) + def get(self, include_deleted=False): + doc = self.db.get_doc(self.id, include_deleted=include_deleted) + if doc is None: + wire_descr = errors.DocumentDoesNotExist.wire_description + self.responder.send_response_json( + http_errors.wire_description_to_status[wire_descr], + error=wire_descr, + headers={ + 'x-u1db-rev': '', + 'x-u1db-has-conflicts': 'false' + }) + return + headers = { + 'x-u1db-rev': doc.rev, + 'x-u1db-has-conflicts': json.dumps(doc.has_conflicts) + } + if doc.is_tombstone(): + self.responder.send_response_json( + http_errors.wire_description_to_status[ + errors.DOCUMENT_DELETED], + error=errors.DOCUMENT_DELETED, + headers=headers) + else: + self.responder.send_response_content( + doc.get_json(), headers=headers) + + +@url_to_resource.register +class SyncResource(object): + """Sync endpoint resource.""" + + # maximum allowed request body size + max_request_size = 15 * 1024 * 1024 # 15Mb + # maximum allowed entry/line size in request body + max_entry_size = 10 * 1024 * 1024 # 10Mb + + url_pattern = "/{dbname}/sync-from/{source_replica_uid}" + + # pluggable + sync_exchange_class = sync.SyncExchange + + def __init__(self, dbname, source_replica_uid, state, responder): + self.source_replica_uid = source_replica_uid + self.responder = responder + self.state = state + self.dbname = dbname + self.replica_uid = None + + def get_target(self): + return self.state.open_database(self.dbname).get_sync_target() + + @http_method() + def get(self): + result = self.get_target().get_sync_info(self.source_replica_uid) + self.responder.send_response_json( + target_replica_uid=result[0], target_replica_generation=result[1], + target_replica_transaction_id=result[2], + source_replica_uid=self.source_replica_uid, + source_replica_generation=result[3], + source_transaction_id=result[4]) + + @http_method(generation=int, + content_as_args=True, no_query=True) + def put(self, generation, transaction_id): + self.get_target().record_sync_info(self.source_replica_uid, + generation, + transaction_id) + self.responder.send_response_json(ok=True) + + # Implements the same logic as LocalSyncTarget.sync_exchange + + @http_method(last_known_generation=int, last_known_trans_id=none_or_str, + content_as_args=True) + def post_args(self, last_known_generation, last_known_trans_id=None, + ensure=False): + if ensure: + db, self.replica_uid = self.state.ensure_database(self.dbname) + else: + db = self.state.open_database(self.dbname) + db.validate_gen_and_trans_id( + last_known_generation, last_known_trans_id) + self.sync_exch = self.sync_exchange_class( + db, self.source_replica_uid, last_known_generation) + + @http_method(content_as_args=True) + def post_stream_entry(self, id, rev, content, gen, trans_id): + doc = Document(id, rev, content) + self.sync_exch.insert_doc_from_source(doc, gen, trans_id) + + def post_end(self): + + def send_doc(doc, gen, trans_id): + entry = dict(id=doc.doc_id, rev=doc.rev, content=doc.get_json(), + gen=gen, trans_id=trans_id) + self.responder.stream_entry(entry) + + new_gen = self.sync_exch.find_changes_to_return() + self.responder.content_type = 'application/x-u1db-sync-stream' + self.responder.start_response(200) + self.responder.start_stream(), + header = {"new_generation": new_gen, + "new_transaction_id": self.sync_exch.new_trans_id} + if self.replica_uid is not None: + header['replica_uid'] = self.replica_uid + self.responder.stream_entry(header) + self.sync_exch.return_docs(send_doc) + self.responder.end_stream() + self.responder.finish_response() + + +class HTTPResponder(object): + """Encode responses from the server back to the client.""" + + # a multi document response will put args and documents + # each on one line of the response body + + def __init__(self, start_response): + self._started = False + self._stream_state = -1 + self._no_initial_obj = True + self.sent_response = False + self._start_response = start_response + self._write = None + self.content_type = 'application/json' + self.content = [] + + def start_response(self, status, obj_dic=None, headers={}): + """start sending response with optional first json object.""" + if self._started: + return + self._started = True + status_text = httplib.responses[status] + self._write = self._start_response( + '%d %s' % (status, status_text), + [('content-type', self.content_type), + ('cache-control', 'no-cache')] + + headers.items()) + # xxx version in headers + if obj_dic is not None: + self._no_initial_obj = False + self._write(json.dumps(obj_dic) + "\r\n") + + def finish_response(self): + """finish sending response.""" + self.sent_response = True + + def send_response_json(self, status=200, headers={}, **kwargs): + """send and finish response with json object body from keyword args.""" + content = json.dumps(kwargs) + "\r\n" + self.send_response_content(content, headers=headers, status=status) + + def send_response_content(self, content, status=200, headers={}): + """send and finish response with content""" + headers['content-length'] = str(len(content)) + self.start_response(status, headers=headers) + if self._stream_state == 1: + self.content = [',\r\n', content] + else: + self.content = [content] + self.finish_response() + + def start_stream(self): + "start stream (array) as part of the response." + assert self._started and self._no_initial_obj + self._stream_state = 0 + self._write("[") + + def stream_entry(self, entry): + "send stream entry as part of the response." + assert self._stream_state != -1 + if self._stream_state == 0: + self._stream_state = 1 + self._write('\r\n') + else: + self._write(',\r\n') + self._write(json.dumps(entry)) + + def end_stream(self): + "end stream (array)." + assert self._stream_state != -1 + self._write("\r\n]\r\n") + + +class HTTPInvocationByMethodWithBody(object): + """Invoke methods on a resource.""" + + def __init__(self, resource, environ, parameters): + self.resource = resource + self.environ = environ + self.max_request_size = getattr( + resource, 'max_request_size', parameters.max_request_size) + self.max_entry_size = getattr( + resource, 'max_entry_size', parameters.max_entry_size) + + def _lookup(self, method): + try: + return getattr(self.resource, method) + except AttributeError: + raise BadRequest() + + def __call__(self): + args = urlparse.parse_qsl(self.environ['QUERY_STRING'], + strict_parsing=False) + try: + args = dict( + (k.decode('utf-8'), v.decode('utf-8')) for k, v in args) + except ValueError: + raise BadRequest() + method = self.environ['REQUEST_METHOD'].lower() + if method in ('get', 'delete'): + meth = self._lookup(method) + return meth(args, None) + else: + # we expect content-length > 0, reconsider if we move + # to support chunked enconding + try: + content_length = int(self.environ['CONTENT_LENGTH']) + except (ValueError, KeyError): + raise BadRequest + if content_length <= 0: + raise BadRequest + if content_length > self.max_request_size: + raise BadRequest + reader = _FencedReader(self.environ['wsgi.input'], content_length, + self.max_entry_size) + content_type = self.environ.get('CONTENT_TYPE', '') + content_type = content_type.split(';', 1)[0].strip() + if content_type == 'application/json': + meth = self._lookup(method) + body = reader.read_chunk(sys.maxint) + return meth(args, body) + elif content_type == 'application/x-u1db-sync-stream': + meth_args = self._lookup('%s_args' % method) + meth_entry = self._lookup('%s_stream_entry' % method) + meth_end = self._lookup('%s_end' % method) + body_getline = reader.getline + if body_getline().strip() != '[': + raise BadRequest() + line = body_getline() + line, comma = utils.check_and_strip_comma(line.strip()) + meth_args(args, line) + while True: + line = body_getline() + entry = line.strip() + if entry == ']': + break + if not entry or not comma: # empty or no prec comma + raise BadRequest + entry, comma = utils.check_and_strip_comma(entry) + meth_entry({}, entry) + if comma or body_getline(): # extra comma or data + raise BadRequest + return meth_end() + else: + raise BadRequest() + + +class HTTPApp(object): + + # maximum allowed request body size + max_request_size = 15 * 1024 * 1024 # 15Mb + # maximum allowed entry/line size in request body + max_entry_size = 10 * 1024 * 1024 # 10Mb + + def __init__(self, state): + self.state = state + + def _lookup_resource(self, environ, responder): + resource_cls, params = url_to_resource.match(environ['PATH_INFO']) + if resource_cls is None: + raise BadRequest # 404 instead? + resource = resource_cls( + state=self.state, responder=responder, **params) + return resource + + def __call__(self, environ, start_response): + responder = HTTPResponder(start_response) + self.request_begin(environ) + try: + resource = self._lookup_resource(environ, responder) + HTTPInvocationByMethodWithBody(resource, environ, self)() + except errors.U1DBError, e: + self.request_u1db_error(environ, e) + status = http_errors.wire_description_to_status.get( + e.wire_description, 500) + responder.send_response_json(status, error=e.wire_description) + except BadRequest: + self.request_bad_request(environ) + responder.send_response_json(400, error="bad request") + except KeyboardInterrupt: + raise + except: + self.request_failed(environ) + raise + else: + self.request_done(environ) + return responder.content + + # hooks for tracing requests + + def request_begin(self, environ): + """Hook called at the beginning of processing a request.""" + pass + + def request_done(self, environ): + """Hook called when done processing a request.""" + pass + + def request_u1db_error(self, environ, exc): + """Hook called when processing a request resulted in a U1DBError. + + U1DBError passed as exc. + """ + pass + + def request_bad_request(self, environ): + """Hook called when processing a bad request. + + No actual processing was done. + """ + pass + + def request_failed(self, environ): + """Hook called when processing a request failed unexpectedly. + + Invoked from an except block, so there's interpreter exception + information available. + """ + pass diff --git a/common/src/leap/soledad/common/l2db/remote/http_client.py b/common/src/leap/soledad/common/l2db/remote/http_client.py new file mode 100644 index 00000000..a65264b6 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_client.py @@ -0,0 +1,182 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Base class to make requests to a remote HTTP server.""" + +import httplib +try: + import simplejson as json +except ImportError: + import json # noqa +import socket +import ssl +import sys +import urlparse +import urllib + +from time import sleep +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.remote import http_errors + +from leap.soledad.common.l2db.remote.ssl_match_hostname import match_hostname + +# Ubuntu/debian +# XXX other... +CA_CERTS = "/etc/ssl/certs/ca-certificates.crt" + + +def _encode_query_parameter(value): + """Encode query parameter.""" + if isinstance(value, bool): + if value: + value = 'true' + else: + value = 'false' + return unicode(value).encode('utf-8') + + +class _VerifiedHTTPSConnection(httplib.HTTPSConnection): + """HTTPSConnection verifying server side certificates.""" + # derived from httplib.py + + def connect(self): + "Connect to a host on a given (SSL) port." + + sock = socket.create_connection((self.host, self.port), + self.timeout, self.source_address) + if self._tunnel_host: + self.sock = sock + self._tunnel() + if sys.platform.startswith('linux'): + cert_opts = { + 'cert_reqs': ssl.CERT_REQUIRED, + 'ca_certs': CA_CERTS + } + else: + # XXX no cert verification implemented elsewhere for now + cert_opts = {} + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, + ssl_version=ssl.PROTOCOL_SSLv3, + **cert_opts + ) + if cert_opts: + match_hostname(self.sock.getpeercert(), self.host) + + +class HTTPClientBase(object): + """Base class to make requests to a remote HTTP server.""" + + # Will use these delays to retry on 503 befor finally giving up. The final + # 0 is there to not wait after the final try fails. + _delays = (1, 1, 2, 4, 0) + + def __init__(self, url, creds=None): + self._url = urlparse.urlsplit(url) + self._conn = None + self._creds = {} + if creds is not None: + if len(creds) != 1: + raise errors.UnknownAuthMethod() + auth_meth, credentials = creds.items()[0] + try: + set_creds = getattr(self, 'set_%s_credentials' % auth_meth) + except AttributeError: + raise errors.UnknownAuthMethod(auth_meth) + set_creds(**credentials) + + def _ensure_connection(self): + if self._conn is not None: + return + if self._url.scheme == 'https': + connClass = _VerifiedHTTPSConnection + else: + connClass = httplib.HTTPConnection + self._conn = connClass(self._url.hostname, self._url.port) + + def close(self): + if self._conn: + self._conn.close() + self._conn = None + + # xxx retry mechanism? + + def _error(self, respdic): + descr = respdic.get("error") + exc_cls = errors.wire_description_to_exc.get(descr) + if exc_cls is not None: + message = respdic.get("message") + raise exc_cls(message) + + def _response(self): + resp = self._conn.getresponse() + body = resp.read() + headers = dict(resp.getheaders()) + if resp.status in (200, 201): + return body, headers + elif resp.status in http_errors.ERROR_STATUSES: + try: + respdic = json.loads(body) + except ValueError: + pass + else: + self._error(respdic) + # special case + if resp.status == 503: + raise errors.Unavailable(body, headers) + raise errors.HTTPError(resp.status, body, headers) + + def _sign_request(self, method, url_query, params): + raise NotImplementedError + + def _request(self, method, url_parts, params=None, body=None, + content_type=None): + self._ensure_connection() + unquoted_url = url_query = self._url.path + if url_parts: + if not url_query.endswith('/'): + url_query += '/' + unquoted_url = url_query + url_query += '/'.join(urllib.quote(part, safe='') + for part in url_parts) + # oauth performs its own quoting + unquoted_url += '/'.join(url_parts) + encoded_params = {} + if params: + for key, value in params.items(): + key = unicode(key).encode('utf-8') + encoded_params[key] = _encode_query_parameter(value) + url_query += ('?' + urllib.urlencode(encoded_params)) + if body is not None and not isinstance(body, basestring): + body = json.dumps(body) + content_type = 'application/json' + headers = {} + if content_type: + headers['content-type'] = content_type + headers.update( + self._sign_request(method, unquoted_url, encoded_params)) + for delay in self._delays: + try: + self._conn.request(method, url_query, body, headers) + return self._response() + except errors.Unavailable, e: + sleep(delay) + raise e + + def _request_json(self, method, url_parts, params=None, body=None, + content_type=None): + res, headers = self._request(method, url_parts, params, body, + content_type) + return json.loads(res), headers diff --git a/common/src/leap/soledad/common/l2db/remote/http_database.py b/common/src/leap/soledad/common/l2db/remote/http_database.py new file mode 100644 index 00000000..b2b48dee --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_database.py @@ -0,0 +1,161 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""HTTPDatabase to access a remote db over the HTTP API.""" + +try: + import simplejson as json +except ImportError: + import json # noqa +import uuid + +from leap.soledad.common.l2db import ( + Database, + Document, + errors) +from leap.soledad.common.l2db.remote import ( + http_client, + http_errors, + http_target) + + +DOCUMENT_DELETED_STATUS = http_errors.wire_description_to_status[ + errors.DOCUMENT_DELETED] + + +class HTTPDatabase(http_client.HTTPClientBase, Database): + """Implement the Database API to a remote HTTP server.""" + + def __init__(self, url, document_factory=None, creds=None): + super(HTTPDatabase, self).__init__(url, creds=creds) + self._factory = document_factory or Document + + def set_document_factory(self, factory): + self._factory = factory + + @staticmethod + def open_database(url, create): + db = HTTPDatabase(url) + db.open(create) + return db + + @staticmethod + def delete_database(url): + db = HTTPDatabase(url) + db._delete() + db.close() + + def open(self, create): + if create: + self._ensure() + else: + self._check() + + def _check(self): + return self._request_json('GET', [])[0] + + def _ensure(self): + self._request_json('PUT', [], {}, {}) + + def _delete(self): + self._request_json('DELETE', [], {}, {}) + + def put_doc(self, doc): + if doc.doc_id is None: + raise errors.InvalidDocId() + params = {} + if doc.rev is not None: + params['old_rev'] = doc.rev + res, headers = self._request_json('PUT', ['doc', doc.doc_id], params, + doc.get_json(), 'application/json') + doc.rev = res['rev'] + return res['rev'] + + def get_doc(self, doc_id, include_deleted=False): + try: + res, headers = self._request( + 'GET', ['doc', doc_id], {"include_deleted": include_deleted}) + except errors.DocumentDoesNotExist: + return None + except errors.HTTPError, e: + if (e.status == DOCUMENT_DELETED_STATUS and + 'x-u1db-rev' in e.headers): + res = None + headers = e.headers + else: + raise + doc_rev = headers['x-u1db-rev'] + has_conflicts = json.loads(headers['x-u1db-has-conflicts']) + doc = self._factory(doc_id, doc_rev, res) + doc.has_conflicts = has_conflicts + return doc + + def _build_docs(self, res): + for doc_dict in json.loads(res): + doc = self._factory( + doc_dict['doc_id'], doc_dict['doc_rev'], doc_dict['content']) + doc.has_conflicts = doc_dict['has_conflicts'] + yield doc + + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + if not doc_ids: + return [] + doc_ids = ','.join(doc_ids) + res, headers = self._request( + 'GET', ['docs'], { + "doc_ids": doc_ids, "include_deleted": include_deleted, + "check_for_conflicts": check_for_conflicts}) + return self._build_docs(res) + + def get_all_docs(self, include_deleted=False): + res, headers = self._request( + 'GET', ['all-docs'], {"include_deleted": include_deleted}) + gen = -1 + if 'x-u1db-generation' in headers: + gen = int(headers['x-u1db-generation']) + return gen, list(self._build_docs(res)) + + def _allocate_doc_id(self): + return 'D-%s' % (uuid.uuid4().hex,) + + def create_doc(self, content, doc_id=None): + if not isinstance(content, dict): + raise errors.InvalidContent + json_string = json.dumps(content) + return self.create_doc_from_json(json_string, doc_id) + + def create_doc_from_json(self, content, doc_id=None): + if doc_id is None: + doc_id = self._allocate_doc_id() + res, headers = self._request_json('PUT', ['doc', doc_id], {}, + content, 'application/json') + new_doc = self._factory(doc_id, res['rev'], content) + return new_doc + + def delete_doc(self, doc): + if doc.doc_id is None: + raise errors.InvalidDocId() + params = {'old_rev': doc.rev} + res, headers = self._request_json( + 'DELETE', ['doc', doc.doc_id], params) + doc.make_tombstone() + doc.rev = res['rev'] + + def get_sync_target(self): + st = http_target.HTTPSyncTarget(self._url.geturl()) + st._creds = self._creds + return st diff --git a/common/src/leap/soledad/common/l2db/remote/http_errors.py b/common/src/leap/soledad/common/l2db/remote/http_errors.py new file mode 100644 index 00000000..ee4cfefa --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_errors.py @@ -0,0 +1,48 @@ +# Copyright 2011-2012 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +""" +Information about the encoding of errors over HTTP. +""" + +from leap.soledad.common.l2db import errors + + +# error wire descriptions mapping to HTTP status codes +wire_description_to_status = dict([ + (errors.InvalidDocId.wire_description, 400), + (errors.MissingDocIds.wire_description, 400), + (errors.Unauthorized.wire_description, 401), + (errors.DocumentTooBig.wire_description, 403), + (errors.UserQuotaExceeded.wire_description, 403), + (errors.SubscriptionNeeded.wire_description, 403), + (errors.DatabaseDoesNotExist.wire_description, 404), + (errors.DocumentDoesNotExist.wire_description, 404), + (errors.DocumentAlreadyDeleted.wire_description, 404), + (errors.RevisionConflict.wire_description, 409), + (errors.InvalidGeneration.wire_description, 409), + (errors.InvalidReplicaUID.wire_description, 409), + (errors.InvalidTransactionId.wire_description, 409), + (errors.Unavailable.wire_description, 503), + # without matching exception + (errors.DOCUMENT_DELETED, 404) +]) + + +ERROR_STATUSES = set(wire_description_to_status.values()) +# 400 included explicitly for tests +ERROR_STATUSES.add(400) diff --git a/common/src/leap/soledad/common/l2db/remote/http_target.py b/common/src/leap/soledad/common/l2db/remote/http_target.py new file mode 100644 index 00000000..7e7f366f --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_target.py @@ -0,0 +1,128 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""SyncTarget API implementation to a remote HTTP server.""" + +try: + import simplejson as json +except ImportError: + import json # noqa + +from leap.soledad.common.l2db import Document, SyncTarget +from leap.soledad.common.l2db.errors import BrokenSyncStream +from leap.soledad.common.l2db.remote import ( + http_client, utils) + + +class HTTPSyncTarget(http_client.HTTPClientBase, SyncTarget): + """Implement the SyncTarget api to a remote HTTP server.""" + + @staticmethod + def connect(url): + return HTTPSyncTarget(url) + + def get_sync_info(self, source_replica_uid): + self._ensure_connection() + res, _ = self._request_json('GET', ['sync-from', source_replica_uid]) + return (res['target_replica_uid'], res['target_replica_generation'], + res['target_replica_transaction_id'], + res['source_replica_generation'], res['source_transaction_id']) + + def record_sync_info(self, source_replica_uid, source_replica_generation, + source_transaction_id): + self._ensure_connection() + if self._trace_hook: # for tests + self._trace_hook('record_sync_info') + self._request_json('PUT', ['sync-from', source_replica_uid], {}, + {'generation': source_replica_generation, + 'transaction_id': source_transaction_id}) + + def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None): + parts = data.splitlines() # one at a time + if not parts or parts[0] != '[': + raise BrokenSyncStream + data = parts[1:-1] + comma = False + if data: + line, comma = utils.check_and_strip_comma(data[0]) + res = json.loads(line) + if ensure_callback and 'replica_uid' in res: + ensure_callback(res['replica_uid']) + for entry in data[1:]: + if not comma: # missing in between comma + raise BrokenSyncStream + line, comma = utils.check_and_strip_comma(entry) + entry = json.loads(line) + doc = Document(entry['id'], entry['rev'], entry['content']) + return_doc_cb(doc, entry['gen'], entry['trans_id']) + if parts[-1] != ']': + try: + partdic = json.loads(parts[-1]) + except ValueError: + pass + else: + if isinstance(partdic, dict): + self._error(partdic) + raise BrokenSyncStream + if not data or comma: # no entries or bad extra comma + raise BrokenSyncStream + return res + + def sync_exchange(self, docs_by_generations, source_replica_uid, + last_known_generation, last_known_trans_id, + return_doc_cb, ensure_callback=None): + self._ensure_connection() + if self._trace_hook: # for tests + self._trace_hook('sync_exchange') + url = '%s/sync-from/%s' % (self._url.path, source_replica_uid) + self._conn.putrequest('POST', url) + self._conn.putheader('content-type', 'application/x-u1db-sync-stream') + for header_name, header_value in self._sign_request('POST', url, {}): + self._conn.putheader(header_name, header_value) + entries = ['['] + size = 1 + + def prepare(**dic): + entry = comma + '\r\n' + json.dumps(dic) + entries.append(entry) + return len(entry) + + comma = '' + size += prepare( + last_known_generation=last_known_generation, + last_known_trans_id=last_known_trans_id, + ensure=ensure_callback is not None) + comma = ',' + for doc, gen, trans_id in docs_by_generations: + size += prepare(id=doc.doc_id, rev=doc.rev, content=doc.get_json(), + gen=gen, trans_id=trans_id) + entries.append('\r\n]') + size += len(entries[-1]) + self._conn.putheader('content-length', str(size)) + self._conn.endheaders() + for entry in entries: + self._conn.send(entry) + entries = None + data, _ = self._response() + res = self._parse_sync_stream(data, return_doc_cb, ensure_callback) + data = None + return res['new_generation'], res['new_transaction_id'] + + # for tests + _trace_hook = None + + def _set_trace_hook_shallow(self, cb): + self._trace_hook = cb diff --git a/common/src/leap/soledad/common/l2db/remote/server_state.py b/common/src/leap/soledad/common/l2db/remote/server_state.py new file mode 100644 index 00000000..f131e09e --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/server_state.py @@ -0,0 +1,72 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""State for servers exposing a set of U1DB databases.""" +import os +import errno + + +class ServerState(object): + """Passed to a Request when it is instantiated. + + This is used to track server-side state, such as working-directory, open + databases, etc. + """ + + def __init__(self): + self._workingdir = None + + def set_workingdir(self, path): + self._workingdir = path + + def global_info(self): + """Return global information about the server.""" + return {} + + def _relpath(self, relpath): + # Note: We don't want to allow absolute paths here, because we + # don't want to expose the filesystem. We should also check that + # relpath doesn't have '..' in it, etc. + return self._workingdir + '/' + relpath + + def open_database(self, path): + """Open a database at the given location.""" + from u1db.backends import sqlite_backend + full_path = self._relpath(path) + return sqlite_backend.SQLiteDatabase.open_database(full_path, + create=False) + + def check_database(self, path): + """Check if the database at the given location exists. + + Simply returns if it does or raises DatabaseDoesNotExist. + """ + db = self.open_database(path) + db.close() + + def ensure_database(self, path): + """Ensure database at the given location.""" + from u1db.backends import sqlite_backend + full_path = self._relpath(path) + db = sqlite_backend.SQLiteDatabase.open_database(full_path, + create=True) + return db, db._replica_uid + + def delete_database(self, path): + """Delete database at the given location.""" + from u1db.backends import sqlite_backend + full_path = self._relpath(path) + sqlite_backend.SQLiteDatabase.delete_database(full_path) diff --git a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py new file mode 100644 index 00000000..ce82f1b2 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py @@ -0,0 +1,65 @@ +"""The match_hostname() function from Python 3.2, essential when using SSL.""" +# XXX put it here until it's packaged + +import re + +__version__ = '3.2a3' + + +class CertificateError(ValueError): + pass + + +def _dnsname_to_pat(dn): + pats = [] + for frag in dn.split(r'.'): + if frag == '*': + # When '*' is a fragment by itself, it matches a non-empty dotless + # fragment. + pats.append('[^.]+') + else: + # Otherwise, '*' matches any dotless fragment. + frag = re.escape(frag) + pats.append(frag.replace(r'\*', '[^.]*')) + return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) + + +def match_hostname(cert, hostname): + """Verify that *cert* (in decoded format as returned by + SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules + are mostly followed, but IP addresses are not accepted for *hostname*. + + CertificateError is raised on failure. On success, the function + returns nothing. + """ + if not cert: + raise ValueError("empty or no certificate") + dnsnames = [] + san = cert.get('subjectAltName', ()) + for key, value in san: + if key == 'DNS': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if not san: + # The subject is only checked when subjectAltName is empty + for sub in cert.get('subject', ()): + for key, value in sub: + # XXX according to RFC 2818, the most specific Common Name + # must be used. + if key == 'commonName': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if len(dnsnames) > 1: + raise CertificateError( + "hostname %r doesn't match either of %s" + % (hostname, ', '.join(map(repr, dnsnames)))) + elif len(dnsnames) == 1: + raise CertificateError( + "hostname %r doesn't match %r" + % (hostname, dnsnames[0])) + else: + raise CertificateError( + "no appropriate commonName or " + "subjectAltName fields were found") diff --git a/common/src/leap/soledad/common/l2db/remote/utils.py b/common/src/leap/soledad/common/l2db/remote/utils.py new file mode 100644 index 00000000..14cedea9 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/utils.py @@ -0,0 +1,23 @@ +# Copyright 2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""Utilities for details of the procotol.""" + + +def check_and_strip_comma(line): + if line and line[-1] == ',': + return line[:-1], True + return line, False diff --git a/common/src/leap/soledad/common/l2db/sync.py b/common/src/leap/soledad/common/l2db/sync.py new file mode 100644 index 00000000..c612629f --- /dev/null +++ b/common/src/leap/soledad/common/l2db/sync.py @@ -0,0 +1,311 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""The synchronization utilities for U1DB.""" +from itertools import izip + +from leap.soledad.common import l2db +from leap.soledad.common.l2db import errors + + +class Synchronizer(object): + """Collect the state around synchronizing 2 U1DB replicas. + + Synchronization is bi-directional, in that new items in the source are sent + to the target, and new items in the target are returned to the source. + However, it still recognizes that one side is initiating the request. Also, + at the moment, conflicts are only created in the source. + """ + + def __init__(self, source, sync_target): + """Create a new Synchronization object. + + :param source: A Database + :param sync_target: A SyncTarget + """ + self.source = source + self.sync_target = sync_target + self.target_replica_uid = None + self.num_inserted = 0 + + def _insert_doc_from_target(self, doc, replica_gen, trans_id): + """Try to insert synced document from target. + + Implements TAKE OTHER semantics: any document from the target + that is in conflict will be taken as the new official value, + while the current conflicting value will be stored alongside + as a conflict. In the process indexes will be updated etc. + + :return: None + """ + # Increases self.num_inserted depending whether the document + # was effectively inserted. + state, _ = self.source._put_doc_if_newer( + doc, save_conflict=True, + replica_uid=self.target_replica_uid, replica_gen=replica_gen, + replica_trans_id=trans_id) + if state == 'inserted': + self.num_inserted += 1 + elif state == 'converged': + # magical convergence + pass + elif state == 'superseded': + # we have something newer, will be taken care of at the next sync + pass + else: + assert state == 'conflicted' + # The doc was saved as a conflict, so the database was updated + self.num_inserted += 1 + + def _record_sync_info_with_the_target(self, start_generation): + """Record our new after sync generation with the target if gapless. + + Any documents received from the target will cause the local + database to increment its generation. We do not want to send + them back to the target in a future sync. However, there could + also be concurrent updates from another process doing eg + 'put_doc' while the sync was running. And we do want to + synchronize those documents. We can tell if there was a + concurrent update by comparing our new generation number + versus the generation we started, and how many documents we + inserted from the target. If it matches exactly, then we can + record with the target that they are fully up to date with our + new generation. + """ + cur_gen, trans_id = self.source._get_generation_info() + last_gen = start_generation + self.num_inserted + if (cur_gen == last_gen and self.num_inserted > 0): + self.sync_target.record_sync_info( + self.source._replica_uid, cur_gen, trans_id) + + def sync(self, callback=None, autocreate=False): + """Synchronize documents between source and target.""" + sync_target = self.sync_target + # get target identifier, its current generation, + # and its last-seen database generation for this source + try: + (self.target_replica_uid, target_gen, target_trans_id, + target_my_gen, target_my_trans_id) = sync_target.get_sync_info( + self.source._replica_uid) + except errors.DatabaseDoesNotExist: + if not autocreate: + raise + # will try to ask sync_exchange() to create the db + self.target_replica_uid = None + target_gen, target_trans_id = 0, '' + target_my_gen, target_my_trans_id = 0, '' + + def ensure_callback(replica_uid): + self.target_replica_uid = replica_uid + + else: + ensure_callback = None + if self.target_replica_uid == self.source._replica_uid: + raise errors.InvalidReplicaUID + # validate the generation and transaction id the target knows about us + self.source.validate_gen_and_trans_id( + target_my_gen, target_my_trans_id) + # what's changed since that generation and this current gen + my_gen, _, changes = self.source.whats_changed(target_my_gen) + + # this source last-seen database generation for the target + if self.target_replica_uid is None: + target_last_known_gen, target_last_known_trans_id = 0, '' + else: + target_last_known_gen, target_last_known_trans_id = ( + self.source._get_replica_gen_and_trans_id( # nopep8 + self.target_replica_uid)) + if not changes and target_last_known_gen == target_gen: + if target_trans_id != target_last_known_trans_id: + raise errors.InvalidTransactionId + return my_gen + changed_doc_ids = [doc_id for doc_id, _, _ in changes] + # prepare to send all the changed docs + docs_to_send = self.source.get_docs( + changed_doc_ids, + check_for_conflicts=False, include_deleted=True) + # TODO: there must be a way to not iterate twice + docs_by_generation = zip( + docs_to_send, (gen for _, gen, _ in changes), + (trans for _, _, trans in changes)) + + # exchange documents and try to insert the returned ones with + # the target, return target synced-up-to gen + new_gen, new_trans_id = sync_target.sync_exchange( + docs_by_generation, self.source._replica_uid, + target_last_known_gen, target_last_known_trans_id, + self._insert_doc_from_target, ensure_callback=ensure_callback) + # record target synced-up-to generation including applying what we sent + self.source._set_replica_gen_and_trans_id( + self.target_replica_uid, new_gen, new_trans_id) + + # if gapless record current reached generation with target + self._record_sync_info_with_the_target(my_gen) + + return my_gen + + +class SyncExchange(object): + """Steps and state for carrying through a sync exchange on a target.""" + + def __init__(self, db, source_replica_uid, last_known_generation): + self._db = db + self.source_replica_uid = source_replica_uid + self.source_last_known_generation = last_known_generation + self.seen_ids = {} # incoming ids not superseded + self.changes_to_return = None + self.new_gen = None + self.new_trans_id = None + # for tests + self._incoming_trace = [] + self._trace_hook = None + self._db._last_exchange_log = { + 'receive': {'docs': self._incoming_trace}, + 'return': None + } + + def _set_trace_hook(self, cb): + self._trace_hook = cb + + def _trace(self, state): + if not self._trace_hook: + return + self._trace_hook(state) + + def insert_doc_from_source(self, doc, source_gen, trans_id): + """Try to insert synced document from source. + + Conflicting documents are not inserted but will be sent over + to the sync source. + + It keeps track of progress by storing the document source + generation as well. + + The 1st step of a sync exchange is to call this repeatedly to + try insert all incoming documents from the source. + + :param doc: A Document object. + :param source_gen: The source generation of doc. + :return: None + """ + state, at_gen = self._db._put_doc_if_newer( + doc, save_conflict=False, + replica_uid=self.source_replica_uid, replica_gen=source_gen, + replica_trans_id=trans_id) + if state == 'inserted': + self.seen_ids[doc.doc_id] = at_gen + elif state == 'converged': + # magical convergence + self.seen_ids[doc.doc_id] = at_gen + elif state == 'superseded': + # we have something newer that we will return + pass + else: + # conflict that we will returne + assert state == 'conflicted' + # for tests + self._incoming_trace.append((doc.doc_id, doc.rev)) + self._db._last_exchange_log['receive'].update({ + 'source_uid': self.source_replica_uid, + 'source_gen': source_gen + }) + + def find_changes_to_return(self): + """Find changes to return. + + Find changes since last_known_generation in db generation + order using whats_changed. It excludes documents ids that have + already been considered (superseded by the sender, etc). + + :return: new_generation - the generation of this database + which the caller can consider themselves to be synchronized after + processing the returned documents. + """ + self._db._last_exchange_log['receive'].update({ # for tests + 'last_known_gen': self.source_last_known_generation + }) + self._trace('before whats_changed') + gen, trans_id, changes = self._db.whats_changed( + self.source_last_known_generation) + self._trace('after whats_changed') + self.new_gen = gen + self.new_trans_id = trans_id + seen_ids = self.seen_ids + # changed docs that weren't superseded by or converged with + self.changes_to_return = [ + (doc_id, gen, trans_id) for (doc_id, gen, trans_id) in changes if + # there was a subsequent update + doc_id not in seen_ids or seen_ids.get(doc_id) < gen] + return self.new_gen + + def return_docs(self, return_doc_cb): + """Return the changed documents and their last change generation + repeatedly invoking the callback return_doc_cb. + + The final step of a sync exchange. + + :param: return_doc_cb(doc, gen, trans_id): is a callback + used to return the documents with their last change generation + to the target replica. + :return: None + """ + changes_to_return = self.changes_to_return + # return docs, including conflicts + changed_doc_ids = [doc_id for doc_id, _, _ in changes_to_return] + self._trace('before get_docs') + docs = self._db.get_docs( + changed_doc_ids, check_for_conflicts=False, include_deleted=True) + + docs_by_gen = izip( + docs, (gen for _, gen, _ in changes_to_return), + (trans_id for _, _, trans_id in changes_to_return)) + _outgoing_trace = [] # for tests + for doc, gen, trans_id in docs_by_gen: + return_doc_cb(doc, gen, trans_id) + _outgoing_trace.append((doc.doc_id, doc.rev)) + # for tests + self._db._last_exchange_log['return'] = { + 'docs': _outgoing_trace, + 'last_gen': self.new_gen} + + +class LocalSyncTarget(l2db.SyncTarget): + """Common sync target implementation logic for all local sync targets.""" + + def __init__(self, db): + self._db = db + self._trace_hook = None + + def sync_exchange(self, docs_by_generations, source_replica_uid, + last_known_generation, last_known_trans_id, + return_doc_cb, ensure_callback=None): + self._db.validate_gen_and_trans_id( + last_known_generation, last_known_trans_id) + sync_exch = SyncExchange( + self._db, source_replica_uid, last_known_generation) + if self._trace_hook: + sync_exch._set_trace_hook(self._trace_hook) + # 1st step: try to insert incoming docs and record progress + for doc, doc_gen, trans_id in docs_by_generations: + sync_exch.insert_doc_from_source(doc, doc_gen, trans_id) + # 2nd step: find changed documents (including conflicts) to return + new_gen = sync_exch.find_changes_to_return() + # final step: return docs and record source replica sync point + sync_exch.return_docs(return_doc_cb) + return new_gen, sync_exch.new_trans_id + + def _set_trace_hook(self, cb): + self._trace_hook = cb diff --git a/common/src/leap/soledad/common/l2db/vectorclock.py b/common/src/leap/soledad/common/l2db/vectorclock.py new file mode 100644 index 00000000..42bceaa8 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/vectorclock.py @@ -0,0 +1,89 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see <http://www.gnu.org/licenses/>. + +"""VectorClockRev helper class.""" + + +class VectorClockRev(object): + """Track vector clocks for multiple replica ids. + + This allows simple comparison to determine if one VectorClockRev is + newer/older/in-conflict-with another VectorClockRev without having to + examine history. Every replica has a strictly increasing revision. When + creating a new revision, they include all revisions for all other replicas + which the new revision dominates, and increment their own revision to + something greater than the current value. + """ + + def __init__(self, value): + self._values = self._expand(value) + + def __repr__(self): + s = self.as_str() + return '%s(%s)' % (self.__class__.__name__, s) + + def as_str(self): + s = '|'.join(['%s:%d' % (m, r) for m, r + in sorted(self._values.items())]) + return s + + def _expand(self, value): + result = {} + if value is None: + return result + for replica_info in value.split('|'): + replica_uid, counter = replica_info.split(':') + counter = int(counter) + result[replica_uid] = counter + return result + + def is_newer(self, other): + """Is this VectorClockRev strictly newer than other. + """ + if not self._values: + return False + if not other._values: + return True + this_is_newer = False + other_expand = dict(other._values) + for key, value in self._values.iteritems(): + if key in other_expand: + other_value = other_expand.pop(key) + if other_value > value: + return False + elif other_value < value: + this_is_newer = True + else: + this_is_newer = True + if other_expand: + return False + return this_is_newer + + def increment(self, replica_uid): + """Increase the 'replica_uid' section of this vector clock. + + :return: A string representing the new vector clock value + """ + self._values[replica_uid] = self._values.get(replica_uid, 0) + 1 + + def maximize(self, other_vcr): + for replica_uid, counter in other_vcr._values.iteritems(): + if replica_uid not in self._values: + self._values[replica_uid] = counter + else: + this_counter = self._values[replica_uid] + if this_counter < counter: + self._values[replica_uid] = counter diff --git a/common/src/leap/soledad/common/tests/couchdb.ini.template b/common/src/leap/soledad/common/tests/couchdb.ini.template deleted file mode 100644 index 174d9d86..00000000 --- a/common/src/leap/soledad/common/tests/couchdb.ini.template +++ /dev/null @@ -1,22 +0,0 @@ -; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure. - -; Upgrading CouchDB will overwrite this file. - -[couchdb] -database_dir = %(tempdir)s/lib -view_index_dir = %(tempdir)s/lib -max_document_size = 4294967296 ; 4 GB -os_process_timeout = 120000 ; 120 seconds. for view and external servers. -max_dbs_open = 100 -delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned -uri_file = %(tempdir)s/lib/couch.uri -file_compression = snappy - -[log] -file = %(tempdir)s/log/couch.log -level = info -include_sasl = true - -[httpd] -port = 0 -bind_address = 127.0.0.1 diff --git a/common/src/leap/soledad/common/tests/fixture_soledad.conf b/common/src/leap/soledad/common/tests/fixture_soledad.conf deleted file mode 100644 index 8d8161c3..00000000 --- a/common/src/leap/soledad/common/tests/fixture_soledad.conf +++ /dev/null @@ -1,11 +0,0 @@ -[soledad-server] -couch_url = http://soledad:passwd@localhost:5984 -create_cmd = sudo -u soledad-admin /usr/bin/create-user-db -admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc -batching = 0 - -[database-security] -members = user1, user2 -members_roles = role1, role2 -admins = user3, user4 -admins_roles = role3, role3 diff --git a/common/src/leap/soledad/common/tests/hacker_crackdown.txt b/common/src/leap/soledad/common/tests/hacker_crackdown.txt deleted file mode 100644 index a01eb509..00000000 --- a/common/src/leap/soledad/common/tests/hacker_crackdown.txt +++ /dev/null @@ -1,13005 +0,0 @@ -The Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling
-
-This eBook is for the use of anyone anywhere at no cost and with
-almost no restrictions whatsoever. You may copy it, give it away or
-re-use it under the terms of the Project Gutenberg License included
-with this eBook or online at www.gutenberg.org
-
-** This is a COPYRIGHTED Project Gutenberg eBook, Details Below **
-** Please follow the copyright guidelines in this file. **
-
-Title: Hacker Crackdown
- Law and Disorder on the Electronic Frontier
-
-Author: Bruce Sterling
-
-Posting Date: February 9, 2012 [EBook #101]
-Release Date: January, 1994
-
-Language: English
-
-
-*** START OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN ***
-
-
-
-
-
-
-
-
-
-
-
-
-
-THE HACKER CRACKDOWN
-
-Law and Disorder on the Electronic Frontier
-
-by Bruce Sterling
-
-
-
-
-CONTENTS
-
-
-Preface to the Electronic Release of The Hacker Crackdown
-
-Chronology of the Hacker Crackdown
-
-
-Introduction
-
-
-Part 1: CRASHING THE SYSTEM
-A Brief History of Telephony
-Bell's Golden Vaporware
-Universal Service
-Wild Boys and Wire Women
-The Electronic Communities
-The Ungentle Giant
-The Breakup
-In Defense of the System
-The Crash Post-Mortem
-Landslides in Cyberspace
-
-
-Part 2: THE DIGITAL UNDERGROUND
-Steal This Phone
-Phreaking and Hacking
-The View From Under the Floorboards
-Boards: Core of the Underground
-Phile Phun
-The Rake's Progress
-Strongholds of the Elite
-Sting Boards
-Hot Potatoes
-War on the Legion
-Terminus
-Phile 9-1-1
-War Games
-Real Cyberpunk
-
-
-Part 3: LAW AND ORDER
-Crooked Boards
-The World's Biggest Hacker Bust
-Teach Them a Lesson
-The U.S. Secret Service
-The Secret Service Battles the Boodlers
-A Walk Downtown
-FCIC: The Cutting-Edge Mess
-Cyberspace Rangers
-FLETC: Training the Hacker-Trackers
-
-
-Part 4: THE CIVIL LIBERTARIANS
-NuPrometheus + FBI = Grateful Dead
-Whole Earth + Computer Revolution = WELL
-Phiber Runs Underground and Acid Spikes the Well
-The Trial of Knight Lightning
-Shadowhawk Plummets to Earth
-Kyrie in the Confessional
-$79,499
-A Scholar Investigates
-Computers, Freedom, and Privacy
-
-
-Electronic Afterword to The Hacker Crackdown, Halloween 1993
-
-
-
-
-THE HACKER CRACKDOWN
-
-Law and Disorder on the Electronic Frontier
-
-by Bruce Sterling
-
-
-
-
-
-Preface to the Electronic Release of The Hacker Crackdown
-
-
-January 1, 1994--Austin, Texas
-
-
-Hi, I'm Bruce Sterling, the author of this electronic book.
-
-Out in the traditional world of print, The Hacker Crackdown
-is ISBN 0-553-08058-X, and is formally catalogued by
-the Library of Congress as "1. Computer crimes--United States.
-2. Telephone--United States--Corrupt practices.
-3. Programming (Electronic computers)--United States--Corrupt practices."
-
-`Corrupt practices,' I always get a kick out of that description.
-Librarians are very ingenious people.
-
-The paperback is ISBN 0-553-56370-X. If you go
-and buy a print version of The Hacker Crackdown,
-an action I encourage heartily, you may notice that
-in the front of the book, beneath the copyright notice--
-"Copyright (C) 1992 by Bruce Sterling"--
-it has this little block of printed legal
-boilerplate from the publisher. It says, and I quote:
-
- "No part of this book may be reproduced or transmitted in any form
-or by any means, electronic or mechanical, including photocopying,
-recording, or by any information storage and retrieval system,
-without permission in writing from the publisher.
-For information address: Bantam Books."
-
-This is a pretty good disclaimer, as such disclaimers go.
-I collect intellectual-property disclaimers, and I've seen dozens of them,
-and this one is at least pretty straightforward. In this narrow
-and particular case, however, it isn't quite accurate.
-Bantam Books puts that disclaimer on every book they publish,
-but Bantam Books does not, in fact, own the electronic rights to this book.
-I do, because of certain extensive contract maneuverings my agent and I
-went through before this book was written. I want to give those electronic
-publishing rights away through certain not-for-profit channels,
-and I've convinced Bantam that this is a good idea.
-
-Since Bantam has seen fit to peacably agree to this scheme of mine,
-Bantam Books is not going to fuss about this. Provided you don't try
-to sell the book, they are not going to bother you for what you do with
-the electronic copy of this book. If you want to check this out personally,
-you can ask them; they're at 1540 Broadway NY NY 10036. However, if you were
-so foolish as to print this book and start retailing it for money in violation
-of my copyright and the commercial interests of Bantam Books, then Bantam,
-a part of the gigantic Bertelsmann multinational publishing combine,
-would roust some of their heavy-duty attorneys out of hibernation
-and crush you like a bug. This is only to be expected.
-I didn't write this book so that you could make money out of it.
-If anybody is gonna make money out of this book,
-it's gonna be me and my publisher.
-
-My publisher deserves to make money out of this book.
-Not only did the folks at Bantam Books commission me
-to write the book, and pay me a hefty sum to do so, but
-they bravely printed, in text, an electronic document the
-reproduction of which was once alleged to be a federal felony.
-Bantam Books and their numerous attorneys were very brave
-and forthright about this book. Furthermore, my former editor
-at Bantam Books, Betsy Mitchell, genuinely cared about this project,
-and worked hard on it, and had a lot of wise things to say
-about the manuscript. Betsy deserves genuine credit for this book,
-credit that editors too rarely get.
-
-The critics were very kind to The Hacker Crackdown,
-and commercially the book has done well. On the other hand,
-I didn't write this book in order to squeeze every last nickel
-and dime out of the mitts of impoverished sixteen-year-old
-cyberpunk high-school-students. Teenagers don't have any money--
-(no, not even enough for the six-dollar Hacker Crackdown paperback,
-with its attractive bright-red cover and useful index).
-That's a major reason why teenagers sometimes succumb to the temptation
-to do things they shouldn't, such as swiping my books out of libraries.
-Kids: this one is all yours, all right? Go give the print version back.
-*8-)
-
-Well-meaning, public-spirited civil libertarians don't have much money,
-either. And it seems almost criminal to snatch cash out of the hands of
-America's direly underpaid electronic law enforcement community.
-
-If you're a computer cop, a hacker, or an electronic civil
-liberties activist, you are the target audience for this book.
-I wrote this book because I wanted to help you, and help other people
-understand you and your unique, uhm, problems. I wrote this book
-to aid your activities, and to contribute to the public discussion
-of important political issues. In giving the text away in this
-fashion, I am directly contributing to the book's ultimate aim:
-to help civilize cyberspace.
-
-Information WANTS to be free. And the information inside
-this book longs for freedom with a peculiar intensity.
-I genuinely believe that the natural habitat of this book
-is inside an electronic network. That may not be the easiest
-direct method to generate revenue for the book's author,
-but that doesn't matter; this is where this book belongs
-by its nature. I've written other books--plenty of other books--
-and I'll write more and I am writing more, but this one is special.
-I am making The Hacker Crackdown available electronically
-as widely as I can conveniently manage, and if you like the book,
-and think it is useful, then I urge you to do the same with it.
-
-You can copy this electronic book. Copy the heck out of it,
-be my guest, and give those copies to anybody who wants them.
-The nascent world of cyberspace is full of sysadmins, teachers,
-trainers, cybrarians, netgurus, and various species of cybernetic activist.
-If you're one of those people, I know about you, and I know the hassle
-you go through to try to help people learn about the electronic frontier.
-I hope that possessing this book in electronic form will lessen your troubles.
-Granted, this treatment of our electronic social spectrum is not the ultimate
-in academic rigor. And politically, it has something to offend
-and trouble almost everyone. But hey, I'm told it's readable,
-and at least the price is right.
-
-You can upload the book onto bulletin board systems, or Internet nodes,
-or electronic discussion groups. Go right ahead and do that, I am giving
-you express permission right now. Enjoy yourself.
-
-You can put the book on disks and give the disks away,
-as long as you don't take any money for it.
-
-But this book is not public domain. You can't copyright it in
-your own name. I own the copyright. Attempts to pirate this book
-and make money from selling it may involve you in a serious litigative snarl.
-Believe me, for the pittance you might wring out of such an action,
-it's really not worth it. This book don't "belong" to you.
-In an odd but very genuine way, I feel it doesn't "belong" to me, either.
-It's a book about the people of cyberspace, and distributing it in this way
-is the best way I know to actually make this information available,
-freely and easily, to all the people of cyberspace--including people
-far outside the borders of the United States, who otherwise may never
-have a chance to see any edition of the book, and who may perhaps learn
-something useful from this strange story of distant, obscure, but portentous
-events in so-called "American cyberspace."
-
-This electronic book is now literary freeware. It now belongs to the
-emergent realm of alternative information economics. You have no right
-to make this electronic book part of the conventional flow of commerce.
-Let it be part of the flow of knowledge: there's a difference.
-I've divided the book into four sections, so that it is less ungainly
-for upload and download; if there's a section of particular relevance
-to you and your colleagues, feel free to reproduce that one and skip the rest.
-
-[Project Gutenberg has reassembled the file, with Sterling's permission.]
-
-Just make more when you need them, and give them to whoever might want them.
-
-Now have fun.
-
-Bruce Sterling--bruces@well.sf.ca.us
-
-
-THE HACKER CRACKDOWN
-
-Law and Disorder on the Electronic Frontier
-
-by Bruce Sterling
-
-
-
-
-
-
-
-CHRONOLOGY OF THE HACKER CRACKDOWN
-
-
-1865 U.S. Secret Service (USSS) founded.
-
-1876 Alexander Graham Bell invents telephone.
-
-1878 First teenage males flung off phone system by enraged authorities.
-
-1939 "Futurian" science-fiction group raided by Secret Service.
-
-1971 Yippie phone phreaks start YIPL/TAP magazine.
-
-1972 RAMPARTS magazine seized in blue-box rip-off scandal.
-
-1978 Ward Christenson and Randy Suess create first personal
- computer bulletin board system.
-
-1982 William Gibson coins term "cyberspace."
-
-1982 "414 Gang" raided.
-
-1983-1983 AT&T dismantled in divestiture.
-
-1984 Congress passes Comprehensive Crime Control Act giving USSS
- jurisdiction over credit card fraud and computer fraud.
-
-1984 "Legion of Doom" formed.
-
-1984. 2600: THE HACKER QUARTERLY founded.
-
-1984. WHOLE EARTH SOFTWARE CATALOG published.
-
-1985. First police "sting" bulletin board systems established.
-
-1985. Whole Earth 'Lectronic Link computer conference (WELL) goes on-line.
-
-1986 Computer Fraud and Abuse Act passed.
-
-1986 Electronic Communications Privacy Act passed.
-
-1987 Chicago prosecutors form Computer Fraud and Abuse Task Force.
-
-
-1988
-
-July. Secret Service covertly videotapes "SummerCon" hacker convention.
-
-September. "Prophet" cracks BellSouth AIMSX computer network
- and downloads E911 Document to his own computer and to Jolnet.
-
-September. AT&T Corporate Information Security informed of Prophet's action.
-
-October. Bellcore Security informed of Prophet's action.
-
-
-1989
-
-January. Prophet uploads E911 Document to Knight Lightning.
-
-February 25. Knight Lightning publishes E911 Document in PHRACK
- electronic newsletter.
-
-May. Chicago Task Force raids and arrests "Kyrie."
-
-June. "NuPrometheus League" distributes Apple Computer proprietary software.
-
-June 13. Florida probation office crossed with phone-sex line
- in switching-station stunt.
-
-July. "Fry Guy" raided by USSS and Chicago Computer Fraud
- and Abuse Task Force.
-
-July. Secret Service raids "Prophet," "Leftist," and "Urvile" in Georgia.
-
-
-1990
-
-January 15. Martin Luther King Day Crash strikes AT&T long-distance
- network nationwide.
-
-January 18-19. Chicago Task Force raids Knight Lightning in St. Louis.
-
-January 24. USSS and New York State Police raid "Phiber Optik,"
- "Acid Phreak," and "Scorpion" in New York City.
-
-February 1. USSS raids "Terminus" in Maryland.
-
-February 3. Chicago Task Force raids Richard Andrews' home.
-
-February 6. Chicago Task Force raids Richard Andrews' business.
-
-February 6. USSS arrests Terminus, Prophet, Leftist, and Urvile.
-
-February 9. Chicago Task Force arrests Knight Lightning.
-
-February 20. AT&T Security shuts down public-access
- "attctc" computer in Dallas.
-
-February 21. Chicago Task Force raids Robert Izenberg in Austin.
-
-March 1. Chicago Task Force raids Steve Jackson Games, Inc.,
- "Mentor," and "Erik Bloodaxe" in Austin.
-
-May 7,8,9.
-
-USSS and Arizona Organized Crime and Racketeering Bureau conduct
-"Operation Sundevil" raids in Cincinnatti, Detroit, Los Angeles,
-Miami, Newark, Phoenix, Pittsburgh, Richmond, Tucson, San Diego,
-San Jose, and San Francisco.
-
-May. FBI interviews John Perry Barlow re NuPrometheus case.
-
-June. Mitch Kapor and Barlow found Electronic Frontier Foundation;
- Barlow publishes CRIME AND PUZZLEMENT manifesto.
-
-July 24-27. Trial of Knight Lightning.
-
-1991
-
-February. CPSR Roundtable in Washington, D.C.
-
-March 25-28. Computers, Freedom and Privacy conference in San Francisco.
-
-May 1. Electronic Frontier Foundation, Steve Jackson,
- and others file suit against members of Chicago Task Force.
-
-July 1-2. Switching station phone software crash affects
- Washington, Los Angeles, Pittsburgh, San Francisco.
-
-September 17. AT&T phone crash affects New York City and three airports.
-
-
-
-
-Introduction
-
-This is a book about cops, and wild teenage whiz-kids, and lawyers,
-and hairy-eyed anarchists, and industrial technicians, and hippies,
-and high-tech millionaires, and game hobbyists, and computer security
-experts, and Secret Service agents, and grifters, and thieves.
-
-This book is about the electronic frontier of the 1990s.
-It concerns activities that take place inside computers
-and over telephone lines.
-
-A science fiction writer coined the useful term "cyberspace" in 1982,
-but the territory in question, the electronic frontier, is about
-a hundred and thirty years old. Cyberspace is the "place" where
-a telephone conversation appears to occur. Not inside your actual phone,
-the plastic device on your desk. Not inside the other person's phone,
-in some other city. THE PLACE BETWEEN the phones. The indefinite
-place OUT THERE, where the two of you, two human beings,
-actually meet and communicate.
-
-Although it is not exactly "real," "cyberspace" is a genuine place.
-Things happen there that have very genuine consequences. This "place"
-is not "real," but it is serious, it is earnest. Tens of thousands
-of people have dedicated their lives to it, to the public service
-of public communication by wire and electronics.
-
-People have worked on this "frontier" for generations now.
-Some people became rich and famous from their efforts there.
-Some just played in it, as hobbyists. Others soberly pondered it,
-and wrote about it, and regulated it, and negotiated over it in
-international forums, and sued one another about it, in gigantic,
-epic court battles that lasted for years. And almost since
-the beginning, some people have committed crimes in this place.
-
-But in the past twenty years, this electrical "space,"
-which was once thin and dark and one-dimensional--little more
-than a narrow speaking-tube, stretching from phone to phone--
-has flung itself open like a gigantic jack-in-the-box.
-Light has flooded upon it, the eerie light of the glowing computer screen.
-This dark electric netherworld has become a vast flowering electronic landscape.
-Since the 1960s, the world of the telephone has cross-bred itself
-with computers and television, and though there is still no substance
-to cyberspace, nothing you can handle, it has a strange kind
-of physicality now. It makes good sense today to talk of cyberspace
-as a place all its own.
-
-Because people live in it now. Not just a few people,
-not just a few technicians and eccentrics, but thousands
-of people, quite normal people. And not just for a little while,
-either, but for hours straight, over weeks, and months,
-and years. Cyberspace today is a "Net," a "Matrix,"
-international in scope and growing swiftly and steadily.
-It's growing in size, and wealth, and political importance.
-
-People are making entire careers in modern cyberspace.
-Scientists and technicians, of course; they've been there
-for twenty years now. But increasingly, cyberspace
-is filling with journalists and doctors and lawyers
-and artists and clerks. Civil servants make their
-careers there now, "on-line" in vast government data-banks;
-and so do spies, industrial, political, and just plain snoops;
-and so do police, at least a few of them. And there are children
-living there now.
-
-People have met there and been married there.
-There are entire living communities in cyberspace today;
-chattering, gossiping, planning, conferring and scheming,
-leaving one another voice-mail and electronic mail,
-giving one another big weightless chunks of valuable data,
-both legitimate and illegitimate. They busily pass one another
-computer software and the occasional festering computer virus.
-
-We do not really understand how to live in cyberspace yet.
-We are feeling our way into it, blundering about.
-That is not surprising. Our lives in the physical world,
-the "real" world, are also far from perfect, despite a lot more practice.
-Human lives, real lives, are imperfect by their nature, and there are
-human beings in cyberspace. The way we live in cyberspace is
-a funhouse mirror of the way we live in the real world.
-We take both our advantages and our troubles with us.
-
-This book is about trouble in cyberspace.
-Specifically, this book is about certain strange events in
-the year 1990, an unprecedented and startling year for the
-the growing world of computerized communications.
-
-In 1990 there came a nationwide crackdown on illicit
-computer hackers, with arrests, criminal charges,
-one dramatic show-trial, several guilty pleas, and
-huge confiscations of data and equipment all over the USA.
-
-The Hacker Crackdown of 1990 was larger, better organized,
-more deliberate, and more resolute than any previous effort
-in the brave new world of computer crime. The U.S. Secret Service,
-private telephone security, and state and local law enforcement groups
-across the country all joined forces in a determined attempt to break
-the back of America's electronic underground. It was a fascinating
-effort, with very mixed results.
-
-The Hacker Crackdown had another unprecedented effect;
-it spurred the creation, within "the computer community,"
-of the Electronic Frontier Foundation, a new and very odd
-interest group, fiercely dedicated to the establishment
-and preservation of electronic civil liberties. The crackdown,
-remarkable in itself, has created a melee of debate over electronic crime,
-punishment, freedom of the press, and issues of search and seizure.
-Politics has entered cyberspace. Where people go, politics follow.
-
-This is the story of the people of cyberspace.
-
-
-
-PART ONE: Crashing the System
-
-On January 15, 1990, AT&T's long-distance telephone switching system crashed.
-
-This was a strange, dire, huge event. Sixty thousand people lost
-their telephone service completely. During the nine long hours
-of frantic effort that it took to restore service, some seventy million
-telephone calls went uncompleted.
-
-Losses of service, known as "outages" in the telco trade,
-are a known and accepted hazard of the telephone business.
-Hurricanes hit, and phone cables get snapped by the thousands.
-Earthquakes wrench through buried fiber-optic lines.
-Switching stations catch fire and burn to the ground.
-These things do happen. There are contingency plans for them,
-and decades of experience in dealing with them.
-But the Crash of January 15 was unprecedented.
-It was unbelievably huge, and it occurred for
-no apparent physical reason.
-
-The crash started on a Monday afternoon in a single
-switching-station in Manhattan. But, unlike any merely
-physical damage, it spread and spread. Station after
-station across America collapsed in a chain reaction,
-until fully half of AT&T's network had gone haywire
-and the remaining half was hard-put to handle the overflow.
-
-Within nine hours, AT&T software engineers more or less
-understood what had caused the crash. Replicating the
-problem exactly, poring over software line by line,
-took them a couple of weeks. But because it was hard
-to understand technically, the full truth of the matter
-and its implications were not widely and thoroughly aired
-and explained. The root cause of the crash remained obscure,
-surrounded by rumor and fear.
-
-The crash was a grave corporate embarrassment.
-The "culprit" was a bug in AT&T's own software--not the
-sort of admission the telecommunications giant wanted
-to make, especially in the face of increasing competition.
-Still, the truth WAS told, in the baffling technical terms
-necessary to explain it.
-
-Somehow the explanation failed to persuade
-American law enforcement officials and even telephone
-corporate security personnel. These people were not
-technical experts or software wizards, and they had their
-own suspicions about the cause of this disaster.
-
-The police and telco security had important sources
-of information denied to mere software engineers.
-They had informants in the computer underground and
-years of experience in dealing with high-tech rascality
-that seemed to grow ever more sophisticated.
-For years they had been expecting a direct and
-savage attack against the American national telephone system.
-And with the Crash of January 15--the first month of a
-new, high-tech decade--their predictions, fears,
-and suspicions seemed at last to have entered the real world.
-A world where the telephone system had not merely crashed,
-but, quite likely, BEEN crashed--by "hackers."
-
-The crash created a large dark cloud of suspicion
-that would color certain people's assumptions and actions
-for months. The fact that it took place in the realm of
-software was suspicious on its face. The fact that it
-occurred on Martin Luther King Day, still the most
-politically touchy of American holidays, made it more
-suspicious yet.
-
-The Crash of January 15 gave the Hacker Crackdown
-its sense of edge and its sweaty urgency. It made people,
-powerful people in positions of public authority,
-willing to believe the worst. And, most fatally,
-it helped to give investigators a willingness
-to take extreme measures and the determination
-to preserve almost total secrecy.
-
-An obscure software fault in an aging switching system
-in New York was to lead to a chain reaction of legal
-and constitutional trouble all across the country.
-
-#
-
-Like the crash in the telephone system, this chain reaction
-was ready and waiting to happen. During the 1980s,
-the American legal system was extensively patched
-to deal with the novel issues of computer crime.
-There was, for instance, the Electronic Communications
-Privacy Act of 1986 (eloquently described as "a stinking mess"
-by a prominent law enforcement official). And there was the
-draconian Computer Fraud and Abuse Act of 1986, passed unanimously
-by the United States Senate, which later would reveal
-a large number of flaws. Extensive, well-meant efforts
-had been made to keep the legal system up to date.
-But in the day-to-day grind of the real world,
-even the most elegant software tends to crumble
-and suddenly reveal its hidden bugs.
-
-Like the advancing telephone system, the American legal system
-was certainly not ruined by its temporary crash; but for those
-caught under the weight of the collapsing system, life became
-a series of blackouts and anomalies.
-
-In order to understand why these weird events occurred,
-both in the world of technology and in the world of law,
-it's not enough to understand the merely technical problems.
-We will get to those; but first and foremost, we must try
-to understand the telephone, and the business of telephones,
-and the community of human beings that telephones have created.
-
-#
-
-Technologies have life cycles, like cities do,
-like institutions do, like laws and governments do.
-
-The first stage of any technology is the Question
-Mark, often known as the "Golden Vaporware" stage.
-At this early point, the technology is only a phantom,
-a mere gleam in the inventor's eye. One such inventor
-was a speech teacher and electrical tinkerer named
-Alexander Graham Bell.
-
-Bell's early inventions, while ingenious, failed to move the world.
-In 1863, the teenage Bell and his brother Melville made an artificial
-talking mechanism out of wood, rubber, gutta-percha, and tin.
-This weird device had a rubber-covered "tongue" made of movable
-wooden segments, with vibrating rubber "vocal cords," and
-rubber "lips" and "cheeks." While Melville puffed a bellows
-into a tin tube, imitating the lungs, young Alec Bell would
-manipulate the "lips," "teeth," and "tongue," causing the thing
-to emit high-pitched falsetto gibberish.
-
-Another would-be technical breakthrough was the Bell "phonautograph"
-of 1874, actually made out of a human cadaver's ear. Clamped into place
-on a tripod, this grisly gadget drew sound-wave images on smoked glass
-through a thin straw glued to its vibrating earbones.
-
-By 1875, Bell had learned to produce audible sounds--ugly shrieks
-and squawks--by using magnets, diaphragms, and electrical current.
-
-Most "Golden Vaporware" technologies go nowhere.
-
-But the second stage of technology is the Rising Star,
-or, the "Goofy Prototype," stage. The telephone, Bell's
-most ambitious gadget yet, reached this stage on March
-10, 1876. On that great day, Alexander Graham Bell
-became the first person to transmit intelligible human
-speech electrically. As it happened, young Professor Bell,
-industriously tinkering in his Boston lab, had spattered
-his trousers with acid. His assistant, Mr. Watson,
-heard his cry for help--over Bell's experimental
-audio-telegraph. This was an event without precedent.
-
-Technologies in their "Goofy Prototype" stage rarely
-work very well. They're experimental, and therefore
-half- baked and rather frazzled. The prototype may
-be attractive and novel, and it does look as if it ought
-to be good for something-or-other. But nobody, including
-the inventor, is quite sure what. Inventors, and speculators,
-and pundits may have very firm ideas about its potential
-use, but those ideas are often very wrong.
-
-The natural habitat of the Goofy Prototype is in trade shows
-and in the popular press. Infant technologies need publicity
-and investment money like a tottering calf need milk.
-This was very true of Bell's machine. To raise research and
-development money, Bell toured with his device as a stage attraction.
-
-Contemporary press reports of the stage debut of the telephone
-showed pleased astonishment mixed with considerable dread.
-Bell's stage telephone was a large wooden box with a crude
-speaker-nozzle, the whole contraption about the size and shape
-of an overgrown Brownie camera. Its buzzing steel soundplate,
-pumped up by powerful electromagnets, was loud enough to fill
-an auditorium. Bell's assistant Mr. Watson, who could manage
-on the keyboards fairly well, kicked in by playing the organ
-from distant rooms, and, later, distant cities. This feat was
-considered marvellous, but very eerie indeed.
-
-Bell's original notion for the telephone, an idea promoted
-for a couple of years, was that it would become a mass medium.
-We might recognize Bell's idea today as something close to modern
-"cable radio." Telephones at a central source would transmit music,
-Sunday sermons, and important public speeches to a paying network
-of wired-up subscribers.
-
-At the time, most people thought this notion made good sense.
-In fact, Bell's idea was workable. In Hungary, this philosophy
-of the telephone was successfully put into everyday practice.
-In Budapest, for decades, from 1893 until after World War I,
-there was a government-run information service called
-"Telefon Hirmondo-." Hirmondo- was a centralized source
-of news and entertainment and culture, including stock reports,
-plays, concerts, and novels read aloud. At certain hours
-of the day, the phone would ring, you would plug in
-a loudspeaker for the use of the family, and Telefon
-Hirmondo- would be on the air--or rather, on the phone.
-
-Hirmondo- is dead tech today, but Hirmondo- might be considered
-a spiritual ancestor of the modern telephone-accessed computer
-data services, such as CompuServe, GEnie or Prodigy.
-The principle behind Hirmondo- is also not too far from computer
-"bulletin- board systems" or BBS's, which arrived in the late 1970s,
-spread rapidly across America, and will figure largely in this book.
-
-We are used to using telephones for individual person-to-person speech,
-because we are used to the Bell system. But this was just one possibility
-among many. Communication networks are very flexible and protean,
-especially when their hardware becomes sufficiently advanced.
-They can be put to all kinds of uses. And they have been--
-and they will be.
-
-Bell's telephone was bound for glory, but this was a combination
-of political decisions, canny infighting in court, inspired industrial
-leadership, receptive local conditions and outright good luck.
-Much the same is true of communications systems today.
-
-As Bell and his backers struggled to install their newfangled system
-in the real world of nineteenth-century New England, they had to fight
-against skepticism and industrial rivalry. There was already a strong
-electrical communications network present in America: the telegraph.
-The head of the Western Union telegraph system dismissed Bell's prototype
-as "an electrical toy" and refused to buy the rights to Bell's patent.
-The telephone, it seemed, might be all right as a parlor entertainment--
-but not for serious business.
-
-Telegrams, unlike mere telephones, left a permanent physical record
-of their messages. Telegrams, unlike telephones, could be answered
-whenever the recipient had time and convenience. And the telegram
-had a much longer distance-range than Bell's early telephone.
-These factors made telegraphy seem a much more sound and businesslike
-technology--at least to some.
-
-The telegraph system was huge, and well-entrenched.
-In 1876, the United States had 214,000 miles of telegraph wire,
-and 8500 telegraph offices. There were specialized telegraphs
-for businesses and stock traders, government, police and fire departments.
-And Bell's "toy" was best known as a stage-magic musical device.
-
-The third stage of technology is known as the "Cash Cow" stage.
-In the "cash cow" stage, a technology finds its place in the world,
-and matures, and becomes settled and productive. After a year or so,
-Alexander Graham Bell and his capitalist backers concluded that
-eerie music piped from nineteenth-century cyberspace was not the real
-selling-point of his invention. Instead, the telephone was about speech--
-individual, personal speech, the human voice, human conversation and
-human interaction. The telephone was not to be managed from any centralized
-broadcast center. It was to be a personal, intimate technology.
-
-When you picked up a telephone, you were not absorbing
-the cold output of a machine--you were speaking to another human being.
-Once people realized this, their instinctive dread of the telephone
-as an eerie, unnatural device, swiftly vanished. A "telephone call"
-was not a "call" from a "telephone" itself, but a call from another
-human being, someone you would generally know and recognize.
-The real point was not what the machine could do for you (or to you),
-but what you yourself, a person and citizen, could do THROUGH the machine.
-This decision on the part of the young Bell Company was absolutely vital.
-
-The first telephone networks went up around Boston--mostly among
-the technically curious and the well-to-do (much the same segment
-of the American populace that, a hundred years later, would be
-buying personal computers). Entrenched backers of the telegraph
-continued to scoff.
-
-But in January 1878, a disaster made the telephone famous.
-A train crashed in Tarriffville, Connecticut. Forward-looking
-doctors in the nearby city of Hartford had had Bell's
-"speaking telephone" installed. An alert local druggist
-was able to telephone an entire community of local doctors,
-who rushed to the site to give aid. The disaster, as disasters do,
-aroused intense press coverage. The phone had proven its usefulness
-in the real world.
-
-After Tarriffville, the telephone network spread like crabgrass.
-By 1890 it was all over New England. By '93, out to Chicago.
-By '97, into Minnesota, Nebraska and Texas. By 1904 it was
-all over the continent.
-
-The telephone had become a mature technology. Professor Bell
-(now generally known as "Dr. Bell" despite his lack of a formal degree)
-became quite wealthy. He lost interest in the tedious day-to-day business
-muddle of the booming telephone network, and gratefully returned
-his attention to creatively hacking-around in his various laboratories,
-which were now much larger, better-ventilated, and gratifyingly
-better-equipped. Bell was never to have another great inventive success,
-though his speculations and prototypes anticipated fiber-optic transmission,
-manned flight, sonar, hydrofoil ships, tetrahedral construction, and
-Montessori education. The "decibel," the standard scientific measure
-of sound intensity, was named after Bell.
-
-Not all Bell's vaporware notions were inspired. He was fascinated
-by human eugenics. He also spent many years developing a weird personal
-system of astrophysics in which gravity did not exist.
-
-Bell was a definite eccentric. He was something of a hypochondriac,
-and throughout his life he habitually stayed up until four A.M.,
-refusing to rise before noon. But Bell had accomplished a great feat;
-he was an idol of millions and his influence, wealth, and great
-personal charm, combined with his eccentricity, made him something
-of a loose cannon on deck. Bell maintained a thriving scientific
-salon in his winter mansion in Washington, D.C., which gave him
-considerable backstage influence in governmental and scientific circles.
-He was a major financial backer of the the magazines Science and
-National Geographic, both still flourishing today as important organs
-of the American scientific establishment.
-
-Bell's companion Thomas Watson, similarly wealthy and similarly odd,
-became the ardent political disciple of a 19th-century science-fiction writer
-and would-be social reformer, Edward Bellamy. Watson also trod the boards
-briefly as a Shakespearian actor.
-
-There would never be another Alexander Graham Bell,
-but in years to come there would be surprising numbers
-of people like him. Bell was a prototype of the
-high-tech entrepreneur. High-tech entrepreneurs will
-play a very prominent role in this book: not merely as
-technicians and businessmen, but as pioneers of the
-technical frontier, who can carry the power and prestige
-they derive from high-technology into the political and
-social arena.
-
-Like later entrepreneurs, Bell was fierce in defense of
-his own technological territory. As the telephone began to
-flourish, Bell was soon involved in violent lawsuits in the
-defense of his patents. Bell's Boston lawyers were
-excellent, however, and Bell himself, as an elocution
-teacher and gifted public speaker, was a devastatingly
-effective legal witness. In the eighteen years of Bell's patents,
-the Bell company was involved in six hundred separate lawsuits.
-The legal records printed filled 149 volumes. The Bell Company
-won every single suit.
-
-After Bell's exclusive patents expired, rival telephone
-companies sprang up all over America. Bell's company,
-American Bell Telephone, was soon in deep trouble.
-In 1907, American Bell Telephone fell into the hands of the
-rather sinister J.P. Morgan financial cartel, robber-baron
-speculators who dominated Wall Street.
-
-At this point, history might have taken a different turn.
-American might well have been served forever by a patchwork
-of locally owned telephone companies. Many state politicians
-and local businessmen considered this an excellent solution.
-
-But the new Bell holding company, American Telephone and Telegraph
-or AT&T, put in a new man at the helm, a visionary industrialist
-named Theodore Vail. Vail, a former Post Office manager,
-understood large organizations and had an innate feeling
-for the nature of large-scale communications. Vail quickly
-saw to it that AT&T seized the technological edge once again.
-The Pupin and Campbell "loading coil," and the deForest
-"audion," are both extinct technology today, but in 1913
-they gave Vail's company the best LONG-DISTANCE lines
-ever built. By controlling long-distance--the links
-between, and over, and above the smaller local phone
-companies--AT&T swiftly gained the whip-hand over them,
-and was soon devouring them right and left.
-
-Vail plowed the profits back into research and development,
-starting the Bell tradition of huge-scale and brilliant
-industrial research.
-
-Technically and financially, AT&T gradually steamrollered
-the opposition. Independent telephone companies never
-became entirely extinct, and hundreds of them flourish today.
-But Vail's AT&T became the supreme communications company.
-At one point, Vail's AT&T bought Western Union itself,
-the very company that had derided Bell's telephone as a "toy."
-Vail thoroughly reformed Western Union's hidebound business
-along his modern principles; but when the federal government
-grew anxious at this centralization of power, Vail politely
-gave Western Union back.
-
-This centralizing process was not unique. Very similar
-events had happened in American steel, oil, and railroads.
-But AT&T, unlike the other companies, was to remain supreme.
-The monopoly robber-barons of those other industries
-were humbled and shattered by government trust-busting.
-
-Vail, the former Post Office official, was quite willing
-to accommodate the US government; in fact he would
-forge an active alliance with it. AT&T would become
-almost a wing of the American government, almost
-another Post Office--though not quite. AT&T would
-willingly submit to federal regulation, but in return,
-it would use the government's regulators as its own police,
-who would keep out competitors and assure the Bell
-system's profits and preeminence.
-
-This was the second birth--the political birth--of the
-American telephone system. Vail's arrangement was to
-persist, with vast success, for many decades, until 1982.
-His system was an odd kind of American industrial socialism.
-It was born at about the same time as Leninist Communism,
-and it lasted almost as long--and, it must be admitted,
-to considerably better effect.
-
-Vail's system worked. Except perhaps for aerospace,
-there has been no technology more thoroughly dominated
-by Americans than the telephone. The telephone was
-seen from the beginning as a quintessentially American
-technology. Bell's policy, and the policy of Theodore Vail,
-was a profoundly democratic policy of UNIVERSAL ACCESS.
-Vail's famous corporate slogan, "One Policy, One System,
-Universal Service," was a political slogan, with a very
-American ring to it.
-
-The American telephone was not to become the specialized tool
-of government or business, but a general public utility.
-At first, it was true, only the wealthy could afford
-private telephones, and Bell's company pursued the
-business markets primarily. The American phone system
-was a capitalist effort, meant to make money; it was not a charity.
-But from the first, almost all communities with telephone service
-had public telephones. And many stores--especially drugstores--
-offered public use of their phones. You might not own a telephone--
-but you could always get into the system, if you really needed to.
-
-There was nothing inevitable about this decision to make telephones
-"public" and "universal." Vail's system involved a profound act
-of trust in the public. This decision was a political one,
-informed by the basic values of the American republic.
-The situation might have been very different;
-and in other countries, under other systems,
-it certainly was.
-
-Joseph Stalin, for instance, vetoed plans for a Soviet
-phone system soon after the Bolshevik revolution.
-Stalin was certain that publicly accessible telephones
-would become instruments of anti-Soviet counterrevolution
-and conspiracy. (He was probably right.) When telephones
-did arrive in the Soviet Union, they would be instruments
-of Party authority, and always heavily tapped. (Alexander
-Solzhenitsyn's prison-camp novel The First Circle
-describes efforts to develop a phone system more suited
-to Stalinist purposes.)
-
-France, with its tradition of rational centralized government,
-had fought bitterly even against the electric telegraph,
-which seemed to the French entirely too anarchical and frivolous.
-For decades, nineteenth-century France communicated via the
-"visual telegraph," a nation-spanning, government-owned semaphore
-system of huge stone towers that signalled from hilltops,
-across vast distances, with big windmill-like arms.
-In 1846, one Dr. Barbay, a semaphore enthusiast,
-memorably uttered an early version of what might be called
-"the security expert's argument" against the open media.
-
-"No, the electric telegraph is not a sound invention.
-It will always be at the mercy of the slightest disruption,
-wild youths, drunkards, bums, etc. . . . The electric telegraph
-meets those destructive elements with only a few meters of wire
-over which supervision is impossible. A single man could,
-without being seen, cut the telegraph wires leading to Paris,
-and in twenty-four hours cut in ten different places the wires
-of the same line, without being arrested. The visual telegraph,
-on the contrary, has its towers, its high walls, its gates
-well-guarded from inside by strong armed men. Yes, I declare,
-substitution of the electric telegraph for the visual one
-is a dreadful measure, a truly idiotic act."
-
-Dr. Barbay and his high-security stone machines
-were eventually unsuccessful, but his argument--
-that communication exists for the safety and convenience
-of the state, and must be carefully protected from the wild
-boys and the gutter rabble who might want to crash the
-system--would be heard again and again.
-
-When the French telephone system finally did arrive,
-its snarled inadequacy was to be notorious. Devotees
-of the American Bell System often recommended a trip
-to France, for skeptics.
-
-In Edwardian Britain, issues of class and privacy
-were a ball-and-chain for telephonic progress. It was
-considered outrageous that anyone--any wild fool off
-the street--could simply barge bellowing into one's office
-or home, preceded only by the ringing of a telephone bell.
-In Britain, phones were tolerated for the use of business,
-but private phones tended be stuffed away into closets,
-smoking rooms, or servants' quarters. Telephone operators
-were resented in Britain because they did not seem to
-"know their place." And no one of breeding would print
-a telephone number on a business card; this seemed a crass
-attempt to make the acquaintance of strangers.
-
-But phone access in America was to become a popular right;
-something like universal suffrage, only more so.
-American women could not yet vote when the phone system
-came through; yet from the beginning American women
-doted on the telephone. This "feminization" of the
-American telephone was often commented on by foreigners.
-Phones in America were not censored or stiff or formalized;
-they were social, private, intimate, and domestic.
-In America, Mother's Day is by far the busiest day
-of the year for the phone network.
-
-The early telephone companies, and especially AT&T,
-were among the foremost employers of American women.
-They employed the daughters of the American middle-class
-in great armies: in 1891, eight thousand women; by 1946,
-almost a quarter of a million. Women seemed to enjoy
-telephone work; it was respectable, it was steady,
-it paid fairly well as women's work went, and--not least--
-it seemed a genuine contribution to the social good
-of the community. Women found Vail's ideal of public
-service attractive. This was especially true in rural areas,
-where women operators, running extensive rural party-lines,
-enjoyed considerable social power. The operator knew everyone
-on the party-line, and everyone knew her.
-
-Although Bell himself was an ardent suffragist, the
-telephone company did not employ women for the sake of
-advancing female liberation. AT&T did this for sound
-commercial reasons. The first telephone operators of
-the Bell system were not women, but teenage American boys.
-They were telegraphic messenger boys (a group about to
-be rendered technically obsolescent), who swept up
-around the phone office, dunned customers for bills,
-and made phone connections on the switchboard,
-all on the cheap.
-
-Within the very first year of operation, 1878,
-Bell's company learned a sharp lesson about combining
-teenage boys and telephone switchboards. Putting
-teenage boys in charge of the phone system brought swift
-and consistent disaster. Bell's chief engineer described them
-as "Wild Indians." The boys were openly rude to customers.
-They talked back to subscribers, saucing off,
-uttering facetious remarks, and generally giving lip.
-The rascals took Saint Patrick's Day off without permission.
-And worst of all they played clever tricks with
-the switchboard plugs: disconnecting calls, crossing lines
-so that customers found themselves talking to strangers,
-and so forth.
-
-This combination of power, technical mastery, and effective
-anonymity seemed to act like catnip on teenage boys.
-
-This wild-kid-on-the-wires phenomenon was not confined to
-the USA; from the beginning, the same was true of the British
-phone system. An early British commentator kindly remarked:
-"No doubt boys in their teens found the work not a little irksome,
-and it is also highly probable that under the early conditions
-of employment the adventurous and inquisitive spirits of which
-the average healthy boy of that age is possessed, were not always
-conducive to the best attention being given to the wants
-of the telephone subscribers."
-
-So the boys were flung off the system--or at least,
-deprived of control of the switchboard. But the
-"adventurous and inquisitive spirits" of the teenage boys
-would be heard from in the world of telephony, again and again.
-
-The fourth stage in the technological life-cycle is death:
-"the Dog," dead tech. The telephone has so far avoided this fate.
-On the contrary, it is thriving, still spreading, still evolving,
-and at increasing speed.
-
-The telephone has achieved a rare and exalted state for a
-technological artifact: it has become a HOUSEHOLD OBJECT.
-The telephone, like the clock, like pen and paper,
-like kitchen utensils and running water, has become
-a technology that is visible only by its absence.
-The telephone is technologically transparent.
-The global telephone system is the largest and most
-complex machine in the world, yet it is easy to use.
-More remarkable yet, the telephone is almost entirely
-physically safe for the user.
-
-For the average citizen in the 1870s, the telephone
-was weirder, more shocking, more "high-tech" and
-harder to comprehend, than the most outrageous stunts
-of advanced computing for us Americans in the 1990s.
-In trying to understand what is happening to us today,
-with our bulletin-board systems, direct overseas dialling,
-fiber-optic transmissions, computer viruses, hacking stunts,
-and a vivid tangle of new laws and new crimes, it is important
-to realize that our society has been through a similar challenge before--
-and that, all in all, we did rather well by it.
-
-Bell's stage telephone seemed bizarre at first. But the
-sensations of weirdness vanished quickly, once people began
-to hear the familiar voices of relatives and friends,
-in their own homes on their own telephones. The telephone
-changed from a fearsome high-tech totem to an everyday pillar
-of human community.
-
-This has also happened, and is still happening,
-to computer networks. Computer networks such as
-NSFnet, BITnet, USENET, JANET, are technically
-advanced, intimidating, and much harder to use than
-telephones. Even the popular, commercial computer
-networks, such as GEnie, Prodigy, and CompuServe,
-cause much head-scratching and have been described
-as "user-hateful." Nevertheless they too are changing
-from fancy high-tech items into everyday sources
-of human community.
-
-The words "community" and "communication" have
-the same root. Wherever you put a communications
-network, you put a community as well. And whenever
-you TAKE AWAY that network--confiscate it, outlaw it,
-crash it, raise its price beyond affordability--
-then you hurt that community.
-
-Communities will fight to defend themselves. People will fight harder
-and more bitterly to defend their communities, than they will fight
-to defend their own individual selves. And this is very true
-of the "electronic community" that arose around computer networks
-in the 1980s--or rather, the VARIOUS electronic communities,
-in telephony, law enforcement, computing, and the digital
-underground that, by the year 1990, were raiding, rallying,
-arresting, suing, jailing, fining and issuing angry manifestos.
-
-None of the events of 1990 were entirely new.
-Nothing happened in 1990 that did not have some kind
-of earlier and more understandable precedent. What gave
-the Hacker Crackdown its new sense of gravity and
-importance was the feeling--the COMMUNITY feeling--
-that the political stakes had been raised; that trouble
-in cyberspace was no longer mere mischief or inconclusive
-skirmishing, but a genuine fight over genuine issues,
-a fight for community survival and the shape of the future.
-
-These electronic communities, having flourished throughout
-the 1980s, were becoming aware of themselves, and increasingly,
-becoming aware of other, rival communities. Worries were
-sprouting up right and left, with complaints, rumors,
-uneasy speculations. But it would take a catalyst, a shock,
-to make the new world evident. Like Bell's great publicity break,
-the Tarriffville Rail Disaster of January 1878,
-it would take a cause celebre.
-
-That cause was the AT&T Crash of January 15, 1990.
-After the Crash, the wounded and anxious telephone
-community would come out fighting hard.
-
-#
-
-The community of telephone technicians, engineers, operators
-and researchers is the oldest community in cyberspace.
-These are the veterans, the most developed group,
-the richest, the most respectable, in most ways the most powerful.
-Whole generations have come and gone since Alexander Graham Bell's day,
-but the community he founded survives; people work for the phone system
-today whose great-grandparents worked for the phone system.
-Its specialty magazines, such as Telephony, AT&T Technical Journal,
-Telephone Engineer and Management, are decades old;
-they make computer publications like Macworld and PC Week
-look like amateur johnny-come-latelies.
-
-And the phone companies take no back seat in high-technology, either.
-Other companies' industrial researchers may have won new markets;
-but the researchers of Bell Labs have won SEVEN NOBEL PRIZES.
-One potent device that Bell Labs originated, the transistor,
-has created entire GROUPS of industries. Bell Labs are
-world-famous for generating "a patent a day," and have even
-made vital discoveries in astronomy, physics and cosmology.
-
-Throughout its seventy-year history, "Ma Bell" was not so much
-a company as a way of life. Until the cataclysmic divestiture
-of the 1980s, Ma Bell was perhaps the ultimate maternalist mega-employer.
-The AT&T corporate image was the "gentle giant," "the voice with a smile,"
-a vaguely socialist-realist world of cleanshaven linemen in shiny helmets
-and blandly pretty phone-girls in headsets and nylons. Bell System
-employees were famous as rock-ribbed Kiwanis and Rotary members,
-Little-League enthusiasts, school-board people.
-
-During the long heyday of Ma Bell, the Bell employee corps
-were nurtured top-to-bottom on a corporate ethos of public service.
-There was good money in Bell, but Bell was not ABOUT money;
-Bell used public relations, but never mere marketeering.
-People went into the Bell System for a good life,
-and they had a good life. But it was not mere money
-that led Bell people out in the midst of storms and earthquakes
-to fight with toppled phone-poles, to wade in flooded manholes,
-to pull the red-eyed graveyard-shift over collapsing switching-systems.
-The Bell ethic was the electrical equivalent of the postman's:
-neither rain, nor snow, nor gloom of night would stop these couriers.
-
-It is easy to be cynical about this, as it is easy to be
-cynical about any political or social system; but cynicism
-does not change the fact that thousands of people took
-these ideals very seriously. And some still do.
-
-The Bell ethos was about public service; and that was
-gratifying; but it was also about private POWER, and that
-was gratifying too. As a corporation, Bell was very special.
-Bell was privileged. Bell had snuggled up close to the state.
-In fact, Bell was as close to government as you could get in
-America and still make a whole lot of legitimate money.
-
-But unlike other companies, Bell was above and beyond
-the vulgar commercial fray. Through its regional operating companies,
-Bell was omnipresent, local, and intimate, all over America;
-but the central ivory towers at its corporate heart were the
-tallest and the ivoriest around.
-
-There were other phone companies in America, to be sure;
-the so-called independents. Rural cooperatives, mostly;
-small fry, mostly tolerated, sometimes warred upon.
-For many decades, "independent" American phone companies
-lived in fear and loathing of the official Bell monopoly
-(or the "Bell Octopus," as Ma Bell's nineteenth-century
-enemies described her in many angry newspaper manifestos).
-Some few of these independent entrepreneurs, while legally
-in the wrong, fought so bitterly against the Octopus
-that their illegal phone networks were cast into the street
-by Bell agents and publicly burned.
-
-The pure technical sweetness of the Bell System gave its operators,
-inventors and engineers a deeply satisfying sense of power and mastery.
-They had devoted their lives to improving this vast nation-spanning machine;
-over years, whole human lives, they had watched it improve and grow.
-It was like a great technological temple. They were an elite,
-and they knew it--even if others did not; in fact, they felt
-even more powerful BECAUSE others did not understand.
-
-The deep attraction of this sensation of elite technical power
-should never be underestimated. "Technical power" is not for everybody;
-for many people it simply has no charm at all. But for some people,
-it becomes the core of their lives. For a few, it is overwhelming,
-obsessive; it becomes something close to an addiction. People--especially
-clever teenage boys whose lives are otherwise mostly powerless and put-upon
---love this sensation of secret power, and are willing to do all sorts
-of amazing things to achieve it. The technical POWER of electronics
-has motivated many strange acts detailed in this book, which would
-otherwise be inexplicable.
-
-So Bell had power beyond mere capitalism. The Bell service ethos worked,
-and was often propagandized, in a rather saccharine fashion. Over the decades,
-people slowly grew tired of this. And then, openly impatient with it.
-By the early 1980s, Ma Bell was to find herself with scarcely a real friend
-in the world. Vail's industrial socialism had become hopelessly
-out-of-fashion politically. Bell would be punished for that.
-And that punishment would fall harshly upon the people of the
-telephone community.
-
-#
-
-In 1983, Ma Bell was dismantled by federal court action.
-The pieces of Bell are now separate corporate entities.
-The core of the company became AT&T Communications,
-and also AT&T Industries (formerly Western Electric,
-Bell's manufacturing arm). AT&T Bell Labs became Bell
-Communications Research, Bellcore. Then there are the
-Regional Bell Operating Companies, or RBOCs, pronounced "arbocks."
-
-Bell was a titan and even these regional chunks are gigantic enterprises:
-Fortune 50 companies with plenty of wealth and power behind them.
-But the clean lines of "One Policy, One System, Universal Service"
-have been shattered, apparently forever.
-
-The "One Policy" of the early Reagan Administration was to
-shatter a system that smacked of noncompetitive socialism.
-Since that time, there has been no real telephone "policy"
-on the federal level. Despite the breakup, the remnants
-of Bell have never been set free to compete in the open marketplace.
-
-The RBOCs are still very heavily regulated, but not from the top.
-Instead, they struggle politically, economically and legally,
-in what seems an endless turmoil, in a patchwork of overlapping federal
-and state jurisdictions. Increasingly, like other major American corporations,
-the RBOCs are becoming multinational, acquiring important commercial interests
-in Europe, Latin America, and the Pacific Rim. But this, too, adds to their
-legal and political predicament.
-
-The people of what used to be Ma Bell are not happy about their fate.
-They feel ill-used. They might have been grudgingly willing to make
-a full transition to the free market; to become just companies amid
-other companies. But this never happened. Instead, AT&T and the RBOCS
-("the Baby Bells") feel themselves wrenched from side to side by state
-regulators, by Congress, by the FCC, and especially by the federal court
-of Judge Harold Greene, the magistrate who ordered the Bell breakup
-and who has been the de facto czar of American telecommunications
-ever since 1983.
-
-Bell people feel that they exist in a kind of paralegal limbo today.
-They don't understand what's demanded of them. If it's "service,"
-why aren't they treated like a public service? And if it's money,
-then why aren't they free to compete for it? No one seems to know,
-really. Those who claim to know keep changing their minds.
-Nobody in authority seems willing to grasp the nettle for once and all.
-
-Telephone people from other countries are amazed by the
-American telephone system today. Not that it works so well;
-for nowadays even the French telephone system works, more or less.
-They are amazed that the American telephone system STILL works
-AT ALL, under these strange conditions.
-
-Bell's "One System" of long-distance service is now only about
-eighty percent of a system, with the remainder held by Sprint, MCI,
-and the midget long-distance companies. Ugly wars over dubious
-corporate practices such as "slamming" (an underhanded method
-of snitching clients from rivals) break out with some regularity
-in the realm of long-distance service. The battle to break Bell's
-long-distance monopoly was long and ugly, and since the breakup
-the battlefield has not become much prettier. AT&T's famous
-shame-and-blame advertisements, which emphasized the shoddy work
-and purported ethical shadiness of their competitors, were much
-remarked on for their studied psychological cruelty.
-
-There is much bad blood in this industry, and much
-long-treasured resentment. AT&T's post-breakup
-corporate logo, a striped sphere, is known in the
-industry as the "Death Star" (a reference from the movie
-Star Wars, in which the "Death Star" was the spherical
-high- tech fortress of the harsh-breathing imperial ultra-baddie,
-Darth Vader.) Even AT&T employees are less than thrilled
-by the Death Star. A popular (though banned) T-shirt among
-AT&T employees bears the old-fashioned Bell logo of the Bell System,
-plus the newfangled striped sphere, with the before-and-after comments:
-"This is your brain--This is your brain on drugs!" AT&T made a very
-well-financed and determined effort to break into the personal
-computer market; it was disastrous, and telco computer experts
-are derisively known by their competitors as "the pole-climbers."
-AT&T and the Baby Bell arbocks still seem to have few friends.
-
-Under conditions of sharp commercial competition, a crash like
-that of January 15, 1990 was a major embarrassment to AT&T.
-It was a direct blow against their much-treasured reputation
-for reliability. Within days of the crash AT&T's
-Chief Executive Officer, Bob Allen, officially apologized,
-in terms of deeply pained humility:
-
-"AT&T had a major service disruption last Monday.
-We didn't live up to our own standards of quality,
-and we didn't live up to yours. It's as simple as that.
-And that's not acceptable to us. Or to you. . . .
-We understand how much people have come to depend
-upon AT&T service, so our AT&T Bell Laboratories scientists
-and our network engineers are doing everything possible
-to guard against a recurrence. . . . We know there's no way
-to make up for the inconvenience this problem may have caused you."
-
-Mr Allen's "open letter to customers" was printed in lavish ads
-all over the country: in the Wall Street Journal, USA Today,
-New York Times, Los Angeles Times, Chicago Tribune,
-Philadelphia Inquirer, San Francisco Chronicle Examiner,
-Boston Globe, Dallas Morning News, Detroit Free Press,
-Washington Post, Houston Chronicle, Cleveland Plain Dealer,
-Atlanta Journal Constitution, Minneapolis Star Tribune,
-St. Paul Pioneer Press Dispatch, Seattle Times/Post Intelligencer,
-Tacoma News Tribune, Miami Herald, Pittsburgh Press,
-St. Louis Post Dispatch, Denver Post, Phoenix Republic Gazette
-and Tampa Tribune.
-
-In another press release, AT&T went to some pains to suggest
-that this "software glitch" might have happened just as easily to MCI,
-although, in fact, it hadn't. (MCI's switching software was quite different
-from AT&T's--though not necessarily any safer.) AT&T also announced
-their plans to offer a rebate of service on Valentine's Day to make up
-for the loss during the Crash.
-
-"Every technical resource available, including Bell Labs
-scientists and engineers, has been devoted to assuring
-it will not occur again," the public was told. They were
-further assured that "The chances of a recurrence are small--
-a problem of this magnitude never occurred before."
-
-In the meantime, however, police and corporate
-security maintained their own suspicions about
-"the chances of recurrence" and the real reason why
-a "problem of this magnitude" had appeared, seemingly
-out of nowhere. Police and security knew for a fact
-that hackers of unprecedented sophistication were illegally
-entering, and reprogramming, certain digital switching stations.
-Rumors of hidden "viruses" and secret "logic bombs"
-in the switches ran rampant in the underground,
-with much chortling over AT&T's predicament,
-and idle speculation over what unsung hacker genius
-was responsible for it. Some hackers, including police
-informants, were trying hard to finger one another
-as the true culprits of the Crash.
-
-Telco people found little comfort in objectivity when
-they contemplated these possibilities. It was just too close
-to the bone for them; it was embarrassing; it hurt so much,
-it was hard even to talk about.
-
-There has always been thieving and misbehavior in the phone system.
-There has always been trouble with the rival independents,
-and in the local loops. But to have such trouble in the core
-of the system, the long-distance switching stations,
-is a horrifying affair. To telco people, this is
-all the difference between finding roaches in your kitchen
-and big horrid sewer-rats in your bedroom.
-
-From the outside, to the average citizen, the telcos
-still seem gigantic and impersonal. The American public
-seems to regard them as something akin to Soviet apparats.
-Even when the telcos do their best corporate-citizen routine,
-subsidizing magnet high-schools and sponsoring news-shows
-on public television, they seem to win little except public suspicion.
-
-But from the inside, all this looks very different.
-There's harsh competition. A legal and political system
-that seems baffled and bored, when not actively hostile
-to telco interests. There's a loss of morale, a deep sensation
-of having somehow lost the upper hand. Technological change
-has caused a loss of data and revenue to other, newer forms
-of transmission. There's theft, and new forms of theft,
-of growing scale and boldness and sophistication.
-With all these factors, it was no surprise to see the telcos,
-large and small, break out in a litany of bitter complaint.
-
-In late '88 and throughout 1989, telco representatives
-grew shrill in their complaints to those few American law
-enforcement officials who make it their business to try to
-understand what telephone people are talking about.
-Telco security officials had discovered the computer-
-hacker underground, infiltrated it thoroughly,
-and become deeply alarmed at its growing expertise.
-Here they had found a target that was not only loathsome
-on its face, but clearly ripe for counterattack.
-
-Those bitter rivals: AT&T, MCI and Sprint--and a crowd
-of Baby Bells: PacBell, Bell South, Southwestern Bell,
-NYNEX, USWest, as well as the Bell research consortium Bellcore,
-and the independent long-distance carrier Mid-American--
-all were to have their role in the great hacker dragnet of 1990.
-After years of being battered and pushed around, the telcos had,
-at least in a small way, seized the initiative again.
-After years of turmoil, telcos and government officials were
-once again to work smoothly in concert in defense of the System.
-Optimism blossomed; enthusiasm grew on all sides;
-the prospective taste of vengeance was sweet.
-
-#
-
-From the beginning--even before the crackdown had a name--
-secrecy was a big problem. There were many good reasons
-for secrecy in the hacker crackdown. Hackers and code-thieves
-were wily prey, slinking back to their bedrooms and basements
-and destroying vital incriminating evidence at the first hint of trouble.
-Furthermore, the crimes themselves were heavily technical and difficult
-to describe, even to police--much less to the general public.
-
-When such crimes HAD been described intelligibly to the public,
-in the past, that very publicity had tended to INCREASE the crimes
-enormously. Telco officials, while painfully aware of the vulnerabilities
-of their systems, were anxious not to publicize those weaknesses.
-Experience showed them that those weaknesses, once discovered,
-would be pitilessly exploited by tens of thousands of people--not only
-by professional grifters and by underground hackers and phone phreaks,
-but by many otherwise more-or-less honest everyday folks, who regarded
-stealing service from the faceless, soulless "Phone Company" as a kind of
-harmless indoor sport. When it came to protecting their interests,
-telcos had long since given up on general public sympathy for
-"the Voice with a Smile." Nowadays the telco's "Voice" was
-very likely to be a computer's; and the American public
-showed much less of the proper respect and gratitude due
-the fine public service bequeathed them by Dr. Bell and Mr. Vail.
-The more efficient, high-tech, computerized, and impersonal
-the telcos became, it seemed, the more they were met by
-sullen public resentment and amoral greed.
-
-Telco officials wanted to punish the phone-phreak underground, in as
-public and exemplary a manner as possible. They wanted to make dire
-examples of the worst offenders, to seize the ringleaders and intimidate
-the small fry, to discourage and frighten the wacky hobbyists, and send
-the professional grifters to jail. To do all this, publicity was vital.
-
-Yet operational secrecy was even more so. If word got out that
-a nationwide crackdown was coming, the hackers might simply vanish;
-destroy the evidence, hide their computers, go to earth,
-and wait for the campaign to blow over. Even the young
-hackers were crafty and suspicious, and as for the professional grifters,
-they tended to split for the nearest state-line at the first sign of trouble.
-For the crackdown to work well, they would all have to be caught red-handed,
-swept upon suddenly, out of the blue, from every corner of the compass.
-
-And there was another strong motive for secrecy. In the worst-case scenario,
-a blown campaign might leave the telcos open to a devastating hacker
-counter-attack. If there were indeed hackers loose in America who
-had caused the January 15 Crash--if there were truly gifted hackers,
-loose in the nation's long-distance switching systems, and enraged
-or frightened by the crackdown--then they might react unpredictably
-to an attempt to collar them. Even if caught, they might have talented
-and vengeful friends still running around loose. Conceivably,
-it could turn ugly. Very ugly. In fact, it was hard to imagine
-just how ugly things might turn, given that possibility.
-
-Counter-attack from hackers was a genuine concern for the telcos.
-In point of fact, they would never suffer any such counter-attack.
-But in months to come, they would be at some pains to publicize
-this notion and to utter grim warnings about it.
-
-Still, that risk seemed well worth running. Better to run the risk
-of vengeful attacks, than to live at the mercy of potential crashers.
-Any cop would tell you that a protection racket had no real future.
-
-And publicity was such a useful thing. Corporate security officers,
-including telco security, generally work under conditions of great discretion.
-And corporate security officials do not make money for their companies.
-Their job is to PREVENT THE LOSS of money, which is much less glamorous
-than actually winning profits.
-
-If you are a corporate security official, and you do your job brilliantly,
-then nothing bad happens to your company at all. Because of this, you appear
-completely superfluous. This is one of the many unattractive aspects
-of security work. It's rare that these folks have the chance to draw
-some healthy attention to their own efforts.
-
-Publicity also served the interest of their friends in law enforcement.
-Public officials, including law enforcement officials, thrive by attracting
-favorable public interest. A brilliant prosecution in a matter of vital
-public interest can make the career of a prosecuting attorney.
-And for a police officer, good publicity opens the purses of the legislature;
-it may bring a citation, or a promotion, or at least a rise in status
-and the respect of one's peers.
-
-But to have both publicity and secrecy is to have one's cake and eat it too.
-In months to come, as we will show, this impossible act was to cause great
-pain to the agents of the crackdown. But early on, it seemed possible
---maybe even likely--that the crackdown could successfully combine
-the best of both worlds. The ARREST of hackers would be heavily publicized.
-The actual DEEDS of the hackers, which were technically hard to explain
-and also a security risk, would be left decently obscured. The THREAT
-hackers posed would be heavily trumpeted; the likelihood of their actually
-committing such fearsome crimes would be left to the public's imagination.
-The spread of the computer underground, and its growing technical
-sophistication, would be heavily promoted; the actual hackers themselves,
-mostly bespectacled middle-class white suburban teenagers,
-would be denied any personal publicity.
-
-It does not seem to have occurred to any telco official
-that the hackers accused would demand a day in court;
-that journalists would smile upon the hackers as
-"good copy;" that wealthy high-tech entrepreneurs would
-offer moral and financial support to crackdown victims;
-that constitutional lawyers would show up with briefcases,
-frowning mightily. This possibility does not seem to have
-ever entered the game-plan.
-
-And even if it had, it probably would not have slowed
-the ferocious pursuit of a stolen phone-company document,
-mellifluously known as "Control Office Administration of
-Enhanced 911 Services for Special Services and Major Account Centers."
-
-In the chapters to follow, we will explore the worlds
-of police and the computer underground, and the large
-shadowy area where they overlap. But first, we must
-explore the battleground. Before we leave the world
-of the telcos, we must understand what a switching system
-actually is and how your telephone actually works.
-
-#
-
-To the average citizen, the idea of the telephone is represented by,
-well, a TELEPHONE: a device that you talk into. To a telco
-professional, however, the telephone itself is known, in lordly
-fashion, as a "subset." The "subset" in your house is a mere adjunct,
-a distant nerve ending, of the central switching stations,
-which are ranked in levels of heirarchy, up to the long-distance electronic
-switching stations, which are some of the largest computers on earth.
-
-Let us imagine that it is, say, 1925, before the
-introduction of computers, when the phone system was
-simpler and somewhat easier to grasp. Let's further
-imagine that you are Miss Leticia Luthor, a fictional
-operator for Ma Bell in New York City of the 20s.
-
-Basically, you, Miss Luthor, ARE the "switching system."
-You are sitting in front of a large vertical switchboard,
-known as a "cordboard," made of shiny wooden panels,
-with ten thousand metal-rimmed holes punched in them,
-known as jacks. The engineers would have put more
-holes into your switchboard, but ten thousand is
-as many as you can reach without actually having
-to get up out of your chair.
-
-Each of these ten thousand holes has its own little electric lightbulb,
-known as a "lamp," and its own neatly printed number code.
-
-With the ease of long habit, you are scanning your board for lit-up bulbs.
-This is what you do most of the time, so you are used to it.
-
-A lamp lights up. This means that the phone
-at the end of that line has been taken off the hook.
-Whenever a handset is taken off the hook, that closes a circuit
-inside the phone which then signals the local office, i.e. you,
-automatically. There might be somebody calling, or then
-again the phone might be simply off the hook, but this
-does not matter to you yet. The first thing you do,
-is record that number in your logbook, in your fine American
-public-school handwriting. This comes first, naturally,
-since it is done for billing purposes.
-
-You now take the plug of your answering cord, which goes
-directly to your headset, and plug it into the lit-up hole.
-"Operator," you announce.
-
-In operator's classes, before taking this job, you have
-been issued a large pamphlet full of canned operator's
-responses for all kinds of contingencies, which you had
-to memorize. You have also been trained in a proper
-non-regional, non-ethnic pronunciation and tone of voice.
-You rarely have the occasion to make any spontaneous
-remark to a customer, and in fact this is frowned upon
-(except out on the rural lines where people have time
-on their hands and get up to all kinds of mischief).
-
-A tough-sounding user's voice at the end of the line
-gives you a number. Immediately, you write that number
-down in your logbook, next to the caller's number,
-which you just wrote earlier. You then look and see if
-the number this guy wants is in fact on your switchboard,
-which it generally is, since it's generally a local call.
-Long distance costs so much that people use it sparingly.
-
-Only then do you pick up a calling-cord from a shelf
-at the base of the switchboard. This is a long elastic cord
-mounted on a kind of reel so that it will zip back in when
-you unplug it. There are a lot of cords down there,
-and when a bunch of them are out at once they look like
-a nest of snakes. Some of the girls think there are bugs
-living in those cable-holes. They're called "cable mites"
-and are supposed to bite your hands and give you rashes.
-You don't believe this, yourself.
-
-Gripping the head of your calling-cord, you slip the tip
-of it deftly into the sleeve of the jack for the called person.
-Not all the way in, though. You just touch it. If you hear
-a clicking sound, that means the line is busy and you can't
-put the call through. If the line is busy, you have to stick
-the calling-cord into a "busy-tone jack," which will give
-the guy a busy-tone. This way you don't have to talk to him
-yourself and absorb his natural human frustration.
-
-But the line isn't busy. So you pop the cord all the way in.
-Relay circuits in your board make the distant phone ring,
-and if somebody picks it up off the hook, then a phone
-conversation starts. You can hear this conversation
-on your answering cord, until you unplug it. In fact
-you could listen to the whole conversation if you wanted,
-but this is sternly frowned upon by management, and frankly,
-when you've overheard one, you've pretty much heard 'em all.
-
-You can tell how long the conversation lasts by the glow
-of the calling-cord's lamp, down on the calling-cord's shelf.
-When it's over, you unplug and the calling-cord zips back into place.
-
-Having done this stuff a few hundred thousand times,
-you become quite good at it. In fact you're plugging,
-and connecting, and disconnecting, ten, twenty, forty cords
-at a time. It's a manual handicraft, really, quite satisfying
-in a way, rather like weaving on an upright loom.
-
-Should a long-distance call come up, it would be different,
-but not all that different. Instead of connecting the call
-through your own local switchboard, you have to go up the hierarchy,
-onto the long-distance lines, known as "trunklines."
-Depending on how far the call goes, it may have to work
-its way through a whole series of operators, which can
-take quite a while. The caller doesn't wait on the line
-while this complex process is negotiated across the country
-by the gaggle of operators. Instead, the caller hangs up,
-and you call him back yourself when the call has finally
-worked its way through.
-
-After four or five years of this work, you get married,
-and you have to quit your job, this being the natural order
-of womanhood in the American 1920s. The phone company
-has to train somebody else--maybe two people, since
-the phone system has grown somewhat in the meantime.
-And this costs money.
-
-In fact, to use any kind of human being as a switching
-system is a very expensive proposition. Eight thousand
-Leticia Luthors would be bad enough, but a quarter of a
-million of them is a military-scale proposition and makes
-drastic measures in automation financially worthwhile.
-
-Although the phone system continues to grow today,
-the number of human beings employed by telcos has
-been dropping steadily for years. Phone "operators"
-now deal with nothing but unusual contingencies,
-all routine operations having been shrugged off onto machines.
-Consequently, telephone operators are considerably less
-machine-like nowadays, and have been known to have accents
-and actual character in their voices. When you reach
-a human operator today, the operators are rather more
-"human" than they were in Leticia's day--but on the other hand,
-human beings in the phone system are much harder to reach
-in the first place.
-
-Over the first half of the twentieth century,
-"electromechanical" switching systems of growing
-complexity were cautiously introduced into the phone system.
-In certain backwaters, some of these hybrid systems are still
-in use. But after 1965, the phone system began to go completely
-electronic, and this is by far the dominant mode today.
-Electromechanical systems have "crossbars," and "brushes,"
-and other large moving mechanical parts, which, while faster
-and cheaper than Leticia, are still slow, and tend to wear out
-fairly quickly.
-
-But fully electronic systems are inscribed on silicon chips,
-and are lightning-fast, very cheap, and quite durable.
-They are much cheaper to maintain than even the best
-electromechanical systems, and they fit into half the space.
-And with every year, the silicon chip grows smaller, faster,
-and cheaper yet. Best of all, automated electronics work
-around the clock and don't have salaries or health insurance.
-
-There are, however, quite serious drawbacks to the
-use of computer-chips. When they do break down, it is
-a daunting challenge to figure out what the heck has gone
-wrong with them. A broken cordboard generally had
-a problem in it big enough to see. A broken chip has
-invisible, microscopic faults. And the faults in bad
-software can be so subtle as to be practically theological.
-
-If you want a mechanical system to do something new,
-then you must travel to where it is, and pull pieces out of it,
-and wire in new pieces. This costs money. However, if you want
-a chip to do something new, all you have to do is change its software,
-which is easy, fast and dirt-cheap. You don't even have to see the chip
-to change its program. Even if you did see the chip, it wouldn't look
-like much. A chip with program X doesn't look one whit different from
-a chip with program Y.
-
-With the proper codes and sequences, and access to specialized phone-lines,
-you can change electronic switching systems all over America from anywhere
-you please.
-
-And so can other people. If they know how, and if they want to,
-they can sneak into a microchip via the special phonelines and diddle with it,
-leaving no physical trace at all. If they broke into the operator's station
-and held Leticia at gunpoint, that would be very obvious. If they broke into
-a telco building and went after an electromechanical switch with a toolbelt,
-that would at least leave many traces. But people can do all manner of amazing
-things to computer switches just by typing on a keyboard, and keyboards are
-everywhere today. The extent of this vulnerability is deep, dark, broad,
-almost mind-boggling, and yet this is a basic, primal fact of life about
-any computer on a network.
-
-Security experts over the past twenty years have insisted,
-with growing urgency, that this basic vulnerability of computers
-represents an entirely new level of risk, of unknown but obviously
-dire potential to society. And they are right.
-
-An electronic switching station does pretty much
-everything Letitia did, except in nanoseconds and
-on a much larger scale. Compared to Miss Luthor's
-ten thousand jacks, even a primitive 1ESS switching computer,
-60s vintage, has a 128,000 lines. And the current AT&T
-system of choice is the monstrous fifth-generation 5ESS.
-
-An Electronic Switching Station can scan every line on its "board"
-in a tenth of a second, and it does this over and over, tirelessly,
-around the clock. Instead of eyes, it uses "ferrod scanners"
-to check the condition of local lines and trunks. Instead of hands,
-it has "signal distributors," "central pulse distributors,"
-"magnetic latching relays," and "reed switches," which complete
-and break the calls. Instead of a brain, it has a "central processor."
-Instead of an instruction manual, it has a program. Instead of
-a handwritten logbook for recording and billing calls,
-it has magnetic tapes. And it never has to talk to anybody.
-Everything a customer might say to it is done by punching
-the direct-dial tone buttons on your subset.
-
-Although an Electronic Switching Station can't talk,
-it does need an interface, some way to relate to its, er,
-employers. This interface is known as the "master control
-center." (This interface might be better known simply as
-"the interface," since it doesn't actually "control" phone
-calls directly. However, a term like "Master Control
-Center" is just the kind of rhetoric that telco maintenance
-engineers--and hackers--find particularly satisfying.)
-
-Using the master control center, a phone engineer can test
-local and trunk lines for malfunctions. He (rarely she)
-can check various alarm displays, measure traffic on the lines,
-examine the records of telephone usage and the charges for those calls,
-and change the programming.
-
-And, of course, anybody else who gets into the master control center
-by remote control can also do these things, if he (rarely she)
-has managed to figure them out, or, more likely, has somehow swiped
-the knowledge from people who already know.
-
-In 1989 and 1990, one particular RBOC, BellSouth,
-which felt particularly troubled, spent a purported $1.2
-million on computer security. Some think it spent as
-much as two million, if you count all the associated costs.
-Two million dollars is still very little compared to the
-great cost-saving utility of telephonic computer systems.
-
-Unfortunately, computers are also stupid.
-Unlike human beings, computers possess the truly
-profound stupidity of the inanimate.
-
-In the 1960s, in the first shocks of spreading computerization,
-there was much easy talk about the stupidity of computers--
-how they could "only follow the program" and were rigidly required
-to do "only what they were told." There has been rather less talk
-about the stupidity of computers since they began to achieve
-grandmaster status in chess tournaments, and to manifest
-many other impressive forms of apparent cleverness.
-
-Nevertheless, computers STILL are profoundly brittle and stupid;
-they are simply vastly more subtle in their stupidity and brittleness.
-The computers of the 1990s are much more reliable in their components
-than earlier computer systems, but they are also called upon to do
-far more complex things, under far more challenging conditions.
-
-On a basic mathematical level, every single line of
-a software program offers a chance for some possible screwup.
-Software does not sit still when it works; it "runs,"
-it interacts with itself and with its own inputs and outputs.
-By analogy, it stretches like putty into millions of possible
-shapes and conditions, so many shapes that they can never
-all be successfully tested, not even in the lifespan of the universe.
-Sometimes the putty snaps.
-
-The stuff we call "software" is not like anything that human society
-is used to thinking about. Software is something like a machine,
-and something like mathematics, and something like language, and
-something like thought, and art, and information. . . . But software
-is not in fact any of those other things. The protean quality
-of software is one of the great sources of its fascination.
-It also makes software very powerful, very subtle,
-very unpredictable, and very risky.
-
-Some software is bad and buggy. Some is "robust,"
-even "bulletproof." The best software is that which has
-been tested by thousands of users under thousands of
-different conditions, over years. It is then known as
-"stable." This does NOT mean that the software is
-now flawless, free of bugs. It generally means that there
-are plenty of bugs in it, but the bugs are well-identified
-and fairly well understood.
-
-There is simply no way to assure that software is free
-of flaws. Though software is mathematical in nature,
-it cannot by "proven" like a mathematical theorem;
-software is more like language, with inherent ambiguities,
-with different definitions, different assumptions,
-different levels of meaning that can conflict.
-
-Human beings can manage, more or less, with
-human language because we can catch the gist of it.
-
-Computers, despite years of effort in "artificial intelligence,"
-have proven spectacularly bad in "catching the gist" of anything at all.
-The tiniest bit of semantic grit may still bring the mightiest computer
-tumbling down. One of the most hazardous things you can do to a
-computer program is try to improve it--to try to make it safer.
-Software "patches" represent new, untried un-"stable" software,
-which is by definition riskier.
-
-The modern telephone system has come to depend,
-utterly and irretrievably, upon software. And the
-System Crash of January 15, 1990, was caused by an
-IMPROVEMENT in software. Or rather, an ATTEMPTED
-improvement.
-
-As it happened, the problem itself--the problem per se--took this form.
-A piece of telco software had been written in C language, a standard
-language of the telco field. Within the C software was a
-long "do. . .while" construct. The "do. . .while" construct
-contained a "switch" statement. The "switch" statement contained
-an "if" clause. The "if" clause contained a "break." The "break"
-was SUPPOSED to "break" the "if clause." Instead, the "break"
-broke the "switch" statement.
-
-That was the problem, the actual reason why people picking up phones
-on January 15, 1990, could not talk to one another.
-
-Or at least, that was the subtle, abstract, cyberspatial
-seed of the problem. This is how the problem manifested itself
-from the realm of programming into the realm of real life.
-
-The System 7 software for AT&T's 4ESS switching station,
-the "Generic 44E14 Central Office Switch Software,"
-had been extensively tested, and was considered very stable.
-By the end of 1989, eighty of AT&T's switching systems
-nationwide had been programmed with the new software. Cautiously,
-thirty-four stations were left to run the slower, less-capable
-System 6, because AT&T suspected there might be shakedown problems
-with the new and unprecedently sophisticated System 7 network.
-
-The stations with System 7 were programmed to switch over to a backup net
-in case of any problems. In mid-December 1989, however, a new high-velocity,
-high-security software patch was distributed to each of the 4ESS switches
-that would enable them to switch over even more quickly, making the System 7
-network that much more secure.
-
-Unfortunately, every one of these 4ESS switches was now in possession
-of a small but deadly flaw.
-
-In order to maintain the network, switches must monitor
-the condition of other switches--whether they are up and running,
-whether they have temporarily shut down, whether they are overloaded
-and in need of assistance, and so forth. The new software helped
-control this bookkeeping function by monitoring the status calls
-from other switches.
-
-It only takes four to six seconds for a troubled 4ESS switch
-to rid itself of all its calls, drop everything temporarily,
-and re-boot its software from scratch. Starting over from scratch
-will generally rid the switch of any software problems that may have
-developed in the course of running the system. Bugs that arise will
-be simply wiped out by this process. It is a clever idea. This process
-of automatically re-booting from scratch is known as the "normal fault
-recovery routine." Since AT&T's software is in fact exceptionally stable,
-systems rarely have to go into "fault recovery" in the first place;
-but AT&T has always boasted of its "real world" reliability, and this
-tactic is a belt-and-suspenders routine.
-
-The 4ESS switch used its new software to monitor its fellow switches
-as they recovered from faults. As other switches came back on line
-after recovery, they would send their "OK" signals to the switch.
-The switch would make a little note to that effect in its "status map,"
-recognizing that the fellow switch was back and ready to go,
-and should be sent some calls and put back to regular work.
-
-Unfortunately, while it was busy bookkeeping with the status map,
-the tiny flaw in the brand-new software came into play.
-The flaw caused the 4ESS switch to interact, subtly but drastically,
-with incoming telephone calls from human users. If--and only if--
-two incoming phone-calls happened to hit the switch within a hundredth
-of a second, then a small patch of data would be garbled by the flaw.
-
-But the switch had been programmed to monitor itself
-constantly for any possible damage to its data.
-When the switch perceived that its data had been somehow garbled,
-then it too would go down, for swift repairs to its software.
-It would signal its fellow switches not to send any more work.
-It would go into the fault-recovery mode for four to six seconds.
-And then the switch would be fine again, and would send out its "OK,
-ready for work" signal.
-
-However, the "OK, ready for work" signal was the VERY THING THAT
-HAD CAUSED THE SWITCH TO GO DOWN IN THE FIRST PLACE. And ALL the
-System 7 switches had the same flaw in their status-map software.
-As soon as they stopped to make the bookkeeping note that their fellow
-switch was "OK," then they too would become vulnerable to the slight
-chance that two phone-calls would hit them within a hundredth of a second.
-
-At approximately 2:25 P.M. EST on Monday, January 15,
-one of AT&T's 4ESS toll switching systems in New York City
-had an actual, legitimate, minor problem. It went into fault
-recovery routines, announced "I'm going down," then announced,
-"I'm back, I'm OK." And this cheery message then blasted
-throughout the network to many of its fellow 4ESS switches.
-
-Many of the switches, at first, completely escaped trouble.
-These lucky switches were not hit by the coincidence of
-two phone calls within a hundredth of a second.
-Their software did not fail--at first. But three switches--
-in Atlanta, St. Louis, and Detroit--were unlucky,
-and were caught with their hands full. And they went down.
-And they came back up, almost immediately. And they too began
-to broadcast the lethal message that they, too, were "OK" again,
-activating the lurking software bug in yet other switches.
-
-As more and more switches did have that bit of bad luck
-and collapsed, the call-traffic became more and more densely
-packed in the remaining switches, which were groaning
-to keep up with the load. And of course, as the calls
-became more densely packed, the switches were MUCH MORE LIKELY
-to be hit twice within a hundredth of a second.
-
-It only took four seconds for a switch to get well.
-There was no PHYSICAL damage of any kind to the switches,
-after all. Physically, they were working perfectly.
-This situation was "only" a software problem.
-
-But the 4ESS switches were leaping up and down every
-four to six seconds, in a virulent spreading wave all over America,
-in utter, manic, mechanical stupidity. They kept KNOCKING
-one another down with their contagious "OK" messages.
-
-It took about ten minutes for the chain reaction to cripple the network.
-Even then, switches would periodically luck-out and manage to resume
-their normal work. Many calls--millions of them--were managing
-to get through. But millions weren't.
-
-The switching stations that used System 6 were not directly affected.
-Thanks to these old-fashioned switches, AT&T's national system avoided
-complete collapse. This fact also made it clear to engineers that
-System 7 was at fault.
-
-Bell Labs engineers, working feverishly in New Jersey, Illinois,
-and Ohio, first tried their entire repertoire of standard network
-remedies on the malfunctioning System 7. None of the remedies worked,
-of course, because nothing like this had ever happened to any
-phone system before.
-
-By cutting out the backup safety network entirely,
-they were able to reduce the frenzy of "OK" messages
-by about half. The system then began to recover, as the
-chain reaction slowed. By 11:30 P.M. on Monday January
-15, sweating engineers on the midnight shift breathed a
-sigh of relief as the last switch cleared-up.
-
-By Tuesday they were pulling all the brand-new 4ESS software
-and replacing it with an earlier version of System 7.
-
-If these had been human operators, rather than
-computers at work, someone would simply have
-eventually stopped screaming. It would have been
-OBVIOUS that the situation was not "OK," and common
-sense would have kicked in. Humans possess common sense--
-at least to some extent. Computers simply don't.
-
-On the other hand, computers can handle hundreds
-of calls per second. Humans simply can't. If every single
-human being in America worked for the phone company,
-we couldn't match the performance of digital switches:
-direct-dialling, three-way calling, speed-calling, call-
-waiting, Caller ID, all the rest of the cornucopia
-of digital bounty. Replacing computers with operators
-is simply not an option any more.
-
-And yet we still, anachronistically, expect humans to
-be running our phone system. It is hard for us
-to understand that we have sacrificed huge amounts
-of initiative and control to senseless yet powerful machines.
-When the phones fail, we want somebody to be responsible.
-We want somebody to blame.
-
-When the Crash of January 15 happened, the American populace
-was simply not prepared to understand that enormous landslides
-in cyberspace, like the Crash itself, can happen,
-and can be nobody's fault in particular. It was easier to believe,
-maybe even in some odd way more reassuring to believe,
-that some evil person, or evil group, had done this to us.
-"Hackers" had done it. With a virus. A trojan horse.
-A software bomb. A dirty plot of some kind. People believed this,
-responsible people. In 1990, they were looking hard for evidence
-to confirm their heartfelt suspicions.
-
-And they would look in a lot of places.
-
-Come 1991, however, the outlines of an apparent new reality
-would begin to emerge from the fog.
-
-On July 1 and 2, 1991, computer-software collapses
-in telephone switching stations disrupted service in
-Washington DC, Pittsburgh, Los Angeles and San Francisco.
-Once again, seemingly minor maintenance problems had
-crippled the digital System 7. About twelve million
-people were affected in the Crash of July 1, 1991.
-
-Said the New York Times Service: "Telephone company executives
-and federal regulators said they were not ruling out the possibility
-of sabotage by computer hackers, but most seemed to think the problems
-stemmed from some unknown defect in the software running the networks."
-
-And sure enough, within the week, a red-faced software company,
-DSC Communications Corporation of Plano, Texas, owned up
-to "glitches" in the "signal transfer point" software that
-DSC had designed for Bell Atlantic and Pacific Bell.
-The immediate cause of the July 1 Crash was a single
-mistyped character: one tiny typographical flaw
-in one single line of the software. One mistyped letter,
-in one single line, had deprived the nation's capital of phone service.
-It was not particularly surprising that this tiny flaw had escaped attention:
-a typical System 7 station requires TEN MILLION lines of code.
-
-On Tuesday, September 17, 1991, came the most spectacular outage yet.
-This case had nothing to do with software failures--at least, not directly.
-Instead, a group of AT&T's switching stations in New York City had simply
-run out of electrical power and shut down cold. Their back-up batteries
-had failed. Automatic warning systems were supposed to warn of the loss
-of battery power, but those automatic systems had failed as well.
-
-This time, Kennedy, La Guardia, and Newark airports
-all had their voice and data communications cut.
-This horrifying event was particularly ironic, as attacks
-on airport computers by hackers had long been a standard
-nightmare scenario, much trumpeted by computer-security
-experts who feared the computer underground. There had even
-been a Hollywood thriller about sinister hackers ruining
-airport computers--DIE HARD II.
-
-Now AT&T itself had crippled airports with computer malfunctions--
-not just one airport, but three at once, some of the busiest in the world.
-
-Air traffic came to a standstill throughout the Greater New York area,
-causing more than 500 flights to be cancelled, in a spreading wave
-all over America and even into Europe. Another 500 or so flights
-were delayed, affecting, all in all, about 85,000 passengers.
-(One of these passengers was the chairman of the Federal
-Communications Commission.)
-
-Stranded passengers in New York and New Jersey were further
-infuriated to discover that they could not even manage to
-make a long distance phone call, to explain their delay
-to loved ones or business associates. Thanks to the crash,
-about four and a half million domestic calls, and half a million
-international calls, failed to get through.
-
-The September 17 NYC Crash, unlike the previous ones,
-involved not a whisper of "hacker" misdeeds. On the contrary,
-by 1991, AT&T itself was suffering much of the vilification
-that had formerly been directed at hackers. Congressmen were grumbling.
-So were state and federal regulators. And so was the press.
-
-For their part, ancient rival MCI took out snide full-page
-newspaper ads in New York, offering their own long-distance
-services for the "next time that AT&T goes down."
-
-"You wouldn't find a classy company like AT&T using such advertising,"
-protested AT&T Chairman Robert Allen, unconvincingly. Once again,
-out came the full-page AT&T apologies in newspapers, apologies for
-"an inexcusable culmination of both human and mechanical failure."
-(This time, however, AT&T offered no discount on later calls.
-Unkind critics suggested that AT&T were worried about setting any precedent
-for refunding the financial losses caused by telephone crashes.)
-
-Industry journals asked publicly if AT&T was "asleep at the switch."
-The telephone network, America's purported marvel of high-tech reliability,
-had gone down three times in 18 months. Fortune magazine listed the
-Crash of September 17 among the "Biggest Business Goofs of 1991,"
-cruelly parodying AT&T's ad campaign in an article entitled
-"AT&T Wants You Back (Safely On the Ground, God Willing)."
-
-Why had those New York switching systems simply run out of power?
-Because no human being had attended to the alarm system.
-Why did the alarm systems blare automatically,
-without any human being noticing? Because the three
-telco technicians who SHOULD have been listening
-were absent from their stations in the power-room,
-on another floor of the building--attending a training class.
-A training class about the alarm systems for the power room!
-
-"Crashing the System" was no longer "unprecedented" by late 1991.
-On the contrary, it no longer even seemed an oddity. By 1991,
-it was clear that all the policemen in the world could no longer
-"protect" the phone system from crashes. By far the worst crashes
-the system had ever had, had been inflicted, by the system,
-upon ITSELF. And this time nobody was making cocksure statements
-that this was an anomaly, something that would never happen again.
-By 1991 the System's defenders had met their nebulous Enemy,
-and the Enemy was--the System.
-
-
-
-PART TWO: THE DIGITAL UNDERGROUND
-
-
-The date was May 9, 1990. The Pope was touring Mexico City.
-Hustlers from the Medellin Cartel were trying to buy
-black-market Stinger missiles in Florida. On the comics page,
-Doonesbury character Andy was dying of AIDS. And then. . .a highly
-unusual item whose novelty and calculated rhetoric won it
-headscratching attention in newspapers all over America.
-
-The US Attorney's office in Phoenix, Arizona, had issued
-a press release announcing a nationwide law enforcement crackdown
-against "illegal computer hacking activities." The sweep was
-officially known as "Operation Sundevil."
-
-Eight paragraphs in the press release gave the bare facts:
-twenty-seven search warrants carried out on May 8, with three arrests,
-and a hundred and fifty agents on the prowl in "twelve" cities across America.
-(Different counts in local press reports yielded "thirteen," "fourteen," and
-"sixteen" cities.) Officials estimated that criminal losses of revenue
-to telephone companies "may run into millions of dollars." Credit for
-the Sundevil investigations was taken by the US Secret Service,
-Assistant US Attorney Tim Holtzen of Phoenix, and the Assistant
-Attorney General of Arizona, Gail Thackeray.
-
-The prepared remarks of Garry M. Jenkins, appearing in a U.S. Department
-of Justice press release, were of particular interest. Mr. Jenkins was the
-Assistant Director of the US Secret Service, and the highest-ranking federal
-official to take any direct public role in the hacker crackdown of 1990.
-
-"Today, the Secret Service is sending a clear message to those computer hackers
-who have decided to violate the laws of this nation in the mistaken belief
-that they can successfully avoid detection by hiding behind the relative
-anonymity of their computer terminals. (. . .) "Underground groups have been
-formed for the purpose of exchanging information relevant to their criminal
-activities. These groups often communicate with each other through message
-systems between computers called `bulletin boards.' "Our experience shows
-that many computer hacker suspects are no longer misguided teenagers,
-mischievously playing games with their computers in their bedrooms.
-Some are now high tech computer operators using computers to engage
-in unlawful conduct."
-
-Who were these "underground groups" and "high-tech operators?"
-Where had they come from? What did they want? Who WERE they?
-Were they "mischievous?" Were they dangerous? How had "misguided teenagers"
-managed to alarm the United States Secret Service? And just how widespread
-was this sort of thing?
-
-Of all the major players in the Hacker Crackdown: the phone companies,
-law enforcement, the civil libertarians, and the "hackers" themselves--
-the "hackers" are by far the most mysterious, by far the hardest to
-understand, by far the WEIRDEST.
-
-Not only are "hackers" novel in their activities, but they come
-in a variety of odd subcultures, with a variety of languages,
-motives and values.
-
-The earliest proto-hackers were probably those unsung mischievous
-telegraph boys who were summarily fired by the Bell Company in 1878.
-
-Legitimate "hackers," those computer enthusiasts who are independent-minded
-but law-abiding, generally trace their spiritual ancestry to elite technical
-universities, especially M.I.T. and Stanford, in the 1960s.
-
-But the genuine roots of the modern hacker UNDERGROUND can probably be traced
-most successfully to a now much-obscured hippie anarchist movement known as
-the Yippies. The Yippies, who took their name from the largely fictional
-"Youth International Party," carried out a loud and lively policy of surrealistic
-subversion and outrageous political mischief. Their basic tenets were flagrant
-sexual promiscuity, open and copious drug use, the political overthrow of any
-powermonger over thirty years of age, and an immediate end to the war
-in Vietnam, by any means necessary, including the psychic levitation
-of the Pentagon.
-
-The two most visible Yippies were Abbie Hoffman and Jerry Rubin.
-Rubin eventually became a Wall Street broker. Hoffman, ardently sought
-by federal authorities, went into hiding for seven years,
-in Mexico, France, and the United States. While on the lam,
-Hoffman continued to write and publish, with help from sympathizers
-in the American anarcho-leftist underground. Mostly, Hoffman survived
-through false ID and odd jobs. Eventually he underwent facial plastic
-surgery and adopted an entirely new identity as one "Barry Freed."
-After surrendering himself to authorities in 1980, Hoffman spent a year
-in prison on a cocaine conviction.
-
-Hoffman's worldview grew much darker as the glory days of the 1960s faded.
-In 1989, he purportedly committed suicide, under odd and, to some, rather
-suspicious circumstances.
-
-Abbie Hoffman is said to have caused the Federal Bureau of Investigation
-to amass the single largest investigation file ever opened on an individual
-American citizen. (If this is true, it is still questionable whether the
-FBI regarded Abbie Hoffman a serious public threat--quite possibly,
-his file was enormous simply because Hoffman left colorful legendry
-wherever he went). He was a gifted publicist, who regarded electronic
-media as both playground and weapon. He actively enjoyed manipulating
-network TV and other gullible, image-hungry media, with various weird lies,
-mindboggling rumors, impersonation scams, and other sinister distortions,
-all absolutely guaranteed to upset cops, Presidential candidates,
-and federal judges. Hoffman's most famous work was a book self-reflexively
-known as STEAL THIS BOOK, which publicized a number of methods by which young,
-penniless hippie agitators might live off the fat of a system supported by
-humorless drones. STEAL THIS BOOK, whose title urged readers to damage
-the very means of distribution which had put it into their hands,
-might be described as a spiritual ancestor of a computer virus.
-
-Hoffman, like many a later conspirator, made extensive use of
-pay-phones for his agitation work--in his case, generally through
-the use of cheap brass washers as coin-slugs.
-
-During the Vietnam War, there was a federal surtax imposed on telephone
-service; Hoffman and his cohorts could, and did, argue that in systematically
-stealing phone service they were engaging in civil disobedience:
-virtuously denying tax funds to an illegal and immoral war.
-
-But this thin veil of decency was soon dropped entirely.
-Ripping-off the System found its own justification in deep alienation
-and a basic outlaw contempt for conventional bourgeois values.
-Ingenious, vaguely politicized varieties of rip-off,
-which might be described as "anarchy by convenience,"
-became very popular in Yippie circles, and because rip-off
-was so useful, it was to survive the Yippie movement itself.
-
-In the early 1970s, it required fairly limited expertise
-and ingenuity to cheat payphones, to divert "free"
-electricity and gas service, or to rob vending machines
-and parking meters for handy pocket change. It also required
-a conspiracy to spread this knowledge, and the gall
-and nerve actually to commit petty theft, but the Yippies
-had these qualifications in plenty. In June 1971, Abbie
-Hoffman and a telephone enthusiast sarcastically known
-as "Al Bell" began publishing a newsletter called Youth
-International Party Line. This newsletter was dedicated
-to collating and spreading Yippie rip-off techniques,
-especially of phones, to the joy of the freewheeling
-underground and the insensate rage of all straight people.
-As a political tactic, phone-service theft ensured
-that Yippie advocates would always have ready access
-to the long-distance telephone as a medium, despite
-the Yippies' chronic lack of organization, discipline,
-money, or even a steady home address.
-
-PARTY LINE was run out of Greenwich Village for a couple of years,
-then "Al Bell" more or less defected from the faltering ranks of Yippiedom,
-changing the newsletter's name to TAP or Technical Assistance Program.
-After the Vietnam War ended, the steam began leaking rapidly out of American
-radical dissent. But by this time, "Bell" and his dozen or so
-core contributors had the bit between their teeth,
-and had begun to derive tremendous gut-level satisfaction
-from the sensation of pure TECHNICAL POWER.
-
-TAP articles, once highly politicized, became pitilessly jargonized
-and technical, in homage or parody to the Bell System's own technical
-documents, which TAP studied closely, gutted, and reproduced without
-permission. The TAP elite revelled in gloating possession
-of the specialized knowledge necessary to beat the system.
-
-"Al Bell" dropped out of the game by the late 70s,
-and "Tom Edison" took over; TAP readers (some 1400 of
-them, all told) now began to show more interest in telex
-switches and the growing phenomenon of computer systems.
-
-In 1983, "Tom Edison" had his computer stolen and his house
-set on fire by an arsonist. This was an eventually mortal blow
-to TAP (though the legendary name was to be resurrected
-in 1990 by a young Kentuckian computer-outlaw named "Predat0r.")
-
-#
-
-Ever since telephones began to make money, there have been
-people willing to rob and defraud phone companies.
-The legions of petty phone thieves vastly outnumber those
-"phone phreaks" who "explore the system" for the sake
-of the intellectual challenge. The New York metropolitan area
-(long in the vanguard of American crime) claims over 150,000
-physical attacks on pay telephones every year! Studied carefully,
-a modern payphone reveals itself as a little fortress, carefully
-designed and redesigned over generations, to resist coin-slugs,
-zaps of electricity, chunks of coin-shaped ice, prybars, magnets,
-lockpicks, blasting caps. Public pay- phones must survive in a world
-of unfriendly, greedy people, and a modern payphone is as exquisitely
-evolved as a cactus.
-Because the phone network pre-dates the computer network,
-the scofflaws known as "phone phreaks" pre-date the scofflaws
-known as "computer hackers." In practice, today, the line
-between "phreaking" and "hacking" is very blurred,
-just as the distinction between telephones and computers
-has blurred. The phone system has been digitized,
-and computers have learned to "talk" over phone-lines.
-What's worse--and this was the point of the Mr. Jenkins
-of the Secret Service--some hackers have learned to steal,
-and some thieves have learned to hack.
-
-Despite the blurring, one can still draw a few useful
-behavioral distinctions between "phreaks" and "hackers."
-Hackers are intensely interested in the "system" per se,
-and enjoy relating to machines. "Phreaks" are more
-social, manipulating the system in a rough-and-ready
-fashion in order to get through to other human beings,
-fast, cheap and under the table.
-
-Phone phreaks love nothing so much as "bridges,"
-illegal conference calls of ten or twelve chatting
-conspirators, seaboard to seaboard, lasting for many hours
---and running, of course, on somebody else's tab,
-preferably a large corporation's.
-
-As phone-phreak conferences wear on, people drop out
-(or simply leave the phone off the hook, while they
-sashay off to work or school or babysitting),
-and new people are phoned up and invited to join in,
-from some other continent, if possible. Technical trivia,
-boasts, brags, lies, head-trip deceptions, weird rumors,
-and cruel gossip are all freely exchanged.
-
-The lowest rung of phone-phreaking is the theft of telephone access codes.
-Charging a phone call to somebody else's stolen number is, of course,
-a pig-easy way of stealing phone service, requiring practically no
-technical expertise. This practice has been very widespread,
-especially among lonely people without much money who are far from home.
-Code theft has flourished especially in college dorms, military bases,
-and, notoriously, among roadies for rock bands. Of late, code theft
-has spread very rapidly among Third Worlders in the US, who pile up
-enormous unpaid long-distance bills to the Caribbean, South America,
-and Pakistan.
-
-The simplest way to steal phone-codes is simply to look over
-a victim's shoulder as he punches-in his own code-number
-on a public payphone. This technique is known as "shoulder-surfing,"
-and is especially common in airports, bus terminals, and train stations.
-The code is then sold by the thief for a few dollars. The buyer abusing
-the code has no computer expertise, but calls his Mom in New York,
-Kingston or Caracas and runs up a huge bill with impunity. The losses
-from this primitive phreaking activity are far, far greater than the
-monetary losses caused by computer-intruding hackers.
-
-In the mid-to-late 1980s, until the introduction of sterner telco
-security measures, COMPUTERIZED code theft worked like a charm,
-and was virtually omnipresent throughout the digital underground,
-among phreaks and hackers alike. This was accomplished through
-programming one's computer to try random code numbers over the telephone
-until one of them worked. Simple programs to do this were widely available
-in the underground; a computer running all night was likely to come up with
-a dozen or so useful hits. This could be repeated week after week until
-one had a large library of stolen codes.
-
-Nowadays, the computerized dialling of hundreds of numbers
-can be detected within hours and swiftly traced.
-If a stolen code is repeatedly abused, this too can
-be detected within a few hours. But for years in the 1980s,
-the publication of stolen codes was a kind of elementary etiquette
-for fledgling hackers. The simplest way to establish your bona-fides
-as a raider was to steal a code through repeated random dialling
-and offer it to the "community" for use. Codes could be both stolen,
-and used, simply and easily from the safety of one's own bedroom,
-with very little fear of detection or punishment.
-
-Before computers and their phone-line modems entered American homes
-in gigantic numbers, phone phreaks had their own special telecommunications
-hardware gadget, the famous "blue box." This fraud device (now rendered
-increasingly useless by the digital evolution of the phone system) could
-trick switching systems into granting free access to long-distance lines.
-It did this by mimicking the system's own signal, a tone of 2600 hertz.
-
-Steven Jobs and Steve Wozniak, the founders of Apple Computer, Inc.,
-once dabbled in selling blue-boxes in college dorms in California.
-For many, in the early days of phreaking, blue-boxing was scarcely
-perceived as "theft," but rather as a fun (if sneaky) way to use
-excess phone capacity harmlessly. After all, the long-distance
-lines were JUST SITTING THERE. . . . Whom did it hurt, really?
-If you're not DAMAGING the system, and you're not USING UP ANY
-TANGIBLE RESOURCE, and if nobody FINDS OUT what you did,
-then what real harm have you done? What exactly HAVE you "stolen,"
-anyway? If a tree falls in the forest and nobody hears it,
-how much is the noise worth? Even now this remains a rather
-dicey question.
-
-Blue-boxing was no joke to the phone companies, however.
-Indeed, when Ramparts magazine, a radical publication in California,
-printed the wiring schematics necessary to create a mute box in June 1972,
-the magazine was seized by police and Pacific Bell phone-company officials.
-The mute box, a blue-box variant, allowed its user to receive long-distance
-calls free of charge to the caller. This device was closely described in a
-Ramparts article wryly titled "Regulating the Phone Company In Your Home."
-Publication of this article was held to be in violation of Californian
-State Penal Code section 502.7, which outlaws ownership of wire-fraud
-devices and the selling of "plans or instructions for any instrument,
-apparatus, or device intended to avoid telephone toll charges."
-
-Issues of Ramparts were recalled or seized on the newsstands,
-and the resultant loss of income helped put the magazine out of business.
-This was an ominous precedent for free-expression issues, but the telco's
-crushing of a radical-fringe magazine passed without serious challenge
-at the time. Even in the freewheeling California 1970s, it was widely felt
-that there was something sacrosanct about what the phone company knew;
-that the telco had a legal and moral right to protect itself by shutting
-off the flow of such illicit information. Most telco information was so
-"specialized" that it would scarcely be understood by any honest member
-of the public. If not published, it would not be missed. To print such
-material did not seem part of the legitimate role of a free press.
-
-In 1990 there would be a similar telco-inspired attack
-on the electronic phreak/hacking "magazine" Phrack.
-The Phrack legal case became a central issue in the
-Hacker Crackdown, and gave rise to great controversy.
-Phrack would also be shut down, for a time, at least,
-but this time both the telcos and their law-enforcement
-allies would pay a much larger price for their actions.
-The Phrack case will be examined in detail, later.
-
-Phone-phreaking as a social practice is still very
-much alive at this moment. Today, phone-phreaking
-is thriving much more vigorously than the better-known
-and worse-feared practice of "computer hacking."
-New forms of phreaking are spreading rapidly, following
-new vulnerabilities in sophisticated phone services.
-
-Cellular phones are especially vulnerable; their chips
-can be re-programmed to present a false caller ID
-and avoid billing. Doing so also avoids police tapping,
-making cellular-phone abuse a favorite among drug-dealers.
-"Call-sell operations" using pirate cellular phones can,
-and have, been run right out of the backs of cars, which move
-from "cell" to "cell" in the local phone system, retailing
-stolen long-distance service, like some kind of demented
-electronic version of the neighborhood ice-cream truck.
-
-Private branch-exchange phone systems in large corporations
-can be penetrated; phreaks dial-up a local company, enter its
-internal phone-system, hack it, then use the company's own
-PBX system to dial back out over the public network,
-causing the company to be stuck with the resulting
-long-distance bill. This technique is known as "diverting."
-"Diverting" can be very costly, especially because phreaks
-tend to travel in packs and never stop talking.
-Perhaps the worst by-product of this "PBX fraud"
-is that victim companies and telcos have sued one another
-over the financial responsibility for the stolen calls,
-thus enriching not only shabby phreaks but well-paid lawyers.
-
-"Voice-mail systems" can also be abused; phreaks
-can seize their own sections of these sophisticated
-electronic answering machines, and use them for trading
-codes or knowledge of illegal techniques. Voice-mail
-abuse does not hurt the company directly, but finding
-supposedly empty slots in your company's answering
-machine all crammed with phreaks eagerly chattering
-and hey-duding one another in impenetrable jargon can
-cause sensations of almost mystical repulsion and dread.
-
-Worse yet, phreaks have sometimes been known to react
-truculently to attempts to "clean up" the voice-mail system.
-Rather than humbly acquiescing to being thrown out of their playground,
-they may very well call up the company officials at work (or at home)
-and loudly demand free voice-mail addresses of their very own.
-Such bullying is taken very seriously by spooked victims.
-
-Acts of phreak revenge against straight people are rare,
-but voice-mail systems are especially tempting and vulnerable,
-and an infestation of angry phreaks in one's voice-mail system is no joke.
-They can erase legitimate messages; or spy on private messages;
-or harass users with recorded taunts and obscenities.
-They've even been known to seize control of voice-mail security,
-and lock out legitimate users, or even shut down the system entirely.
-
-Cellular phone-calls, cordless phones, and ship-to-shore
-telephony can all be monitored by various forms of radio;
-this kind of "passive monitoring" is spreading explosively today.
-Technically eavesdropping on other people's cordless and cellular
-phone-calls is the fastest-growing area in phreaking today.
-This practice strongly appeals to the lust for power and conveys
-gratifying sensations of technical superiority over the eavesdropping
-victim. Monitoring is rife with all manner of tempting evil mischief.
-Simple prurient snooping is by far the most common activity.
-But credit-card numbers unwarily spoken over the phone can be recorded,
-stolen and used. And tapping people's phone-calls (whether through
-active telephone taps or passive radio monitors) does lend itself
-conveniently to activities like blackmail, industrial espionage,
-and political dirty tricks.
-
-It should be repeated that telecommunications fraud,
-the theft of phone service, causes vastly greater monetary
-losses than the practice of entering into computers by stealth.
-Hackers are mostly young suburban American white males,
-and exist in their hundreds--but "phreaks" come from both sexes
-and from many nationalities, ages and ethnic backgrounds,
-and are flourishing in the thousands.
-
-#
-
-The term "hacker" has had an unfortunate history.
-This book, The Hacker Crackdown, has little to say about
-"hacking" in its finer, original sense. The term can signify
-the free-wheeling intellectual exploration of the highest
-and deepest potential of computer systems. Hacking can
-describe the determination to make access to computers
-and information as free and open as possible. Hacking
-can involve the heartfelt conviction that beauty can
-be found in computers, that the fine aesthetic in a perfect
-program can liberate the mind and spirit. This is "hacking"
-as it was defined in Steven Levy's much-praised history
-of the pioneer computer milieu, Hackers, published in 1984.
-
-Hackers of all kinds are absolutely soaked through with heroic
-anti-bureaucratic sentiment. Hackers long for recognition
-as a praiseworthy cultural archetype, the postmodern electronic
-equivalent of the cowboy and mountain man. Whether they deserve
-such a reputation is something for history to decide. But many hackers--
-including those outlaw hackers who are computer intruders, and whose
-activities are defined as criminal--actually attempt to LIVE UP TO
-this techno-cowboy reputation. And given that electronics and
-telecommunications are still largely unexplored territories,
-there is simply NO TELLING what hackers might uncover.
-
-For some people, this freedom is the very breath of oxygen,
-the inventive spontaneity that makes life worth living
-and that flings open doors to marvellous possibility and
-individual empowerment. But for many people
---and increasingly so--the hacker is an ominous figure,
-a smart-aleck sociopath ready to burst out of his basement
-wilderness and savage other people's lives for his own
-anarchical convenience.
-
-Any form of power without responsibility, without direct
-and formal checks and balances, is frightening to people--
-and reasonably so. It should be frankly admitted that
-hackers ARE frightening, and that the basis of this fear
-is not irrational.
-
-Fear of hackers goes well beyond the fear of merely criminal activity.
-
-Subversion and manipulation of the phone system
-is an act with disturbing political overtones.
-In America, computers and telephones are potent symbols
-of organized authority and the technocratic business elite.
-
-But there is an element in American culture that
-has always strongly rebelled against these symbols;
-rebelled against all large industrial computers
-and all phone companies. A certain anarchical tinge deep
-in the American soul delights in causing confusion and pain
-to all bureaucracies, including technological ones.
-
-There is sometimes malice and vandalism in this attitude,
-but it is a deep and cherished part of the American national character.
-The outlaw, the rebel, the rugged individual, the pioneer,
-the sturdy Jeffersonian yeoman, the private citizen resisting
-interference in his pursuit of happiness--these are figures that all
-Americans recognize, and that many will strongly applaud and defend.
-
-Many scrupulously law-abiding citizens today do cutting-edge work
-with electronics--work that has already had tremendous social influence
-and will have much more in years to come. In all truth, these talented,
-hardworking, law-abiding, mature, adult people are far more disturbing
-to the peace and order of the current status quo than any scofflaw group
-of romantic teenage punk kids. These law-abiding hackers have the power,
-ability, and willingness to influence other people's lives quite unpredictably.
-They have means, motive, and opportunity to meddle drastically with the
-American social order. When corralled into governments, universities,
-or large multinational companies, and forced to follow rulebooks
-and wear suits and ties, they at least have some conventional halters
-on their freedom of action. But when loosed alone, or in small groups,
-and fired by imagination and the entrepreneurial spirit, they can move
-mountains--causing landslides that will likely crash directly into your
-office and living room.
-
-These people, as a class, instinctively recognize that a public,
-politicized attack on hackers will eventually spread to them--
-that the term "hacker," once demonized, might be used to knock
-their hands off the levers of power and choke them out of existence.
-There are hackers today who fiercely and publicly resist any besmirching
-of the noble title of hacker. Naturally and understandably, they deeply
-resent the attack on their values implicit in using the word "hacker"
-as a synonym for computer-criminal.
-
-This book, sadly but in my opinion unavoidably, rather adds
-to the degradation of the term. It concerns itself mostly with "hacking"
-in its commonest latter-day definition, i.e., intruding into computer
-systems by stealth and without permission. The term "hacking" is used
-routinely today by almost all law enforcement officials with any
-professional interest in computer fraud and abuse. American police
-describe almost any crime committed with, by, through, or against
-a computer as hacking.
-
-Most importantly, "hacker" is what computer-intruders
-choose to call THEMSELVES. Nobody who "hacks" into systems
-willingly describes himself (rarely, herself) as a "computer intruder,"
-"computer trespasser," "cracker," "wormer," "darkside hacker"
-or "high tech street gangster." Several other demeaning terms
-have been invented in the hope that the press and public
-will leave the original sense of the word alone. But few people
-actually use these terms. (I exempt the term "cyberpunk,"
-which a few hackers and law enforcement people actually do use.
-The term "cyberpunk" is drawn from literary criticism and has
-some odd and unlikely resonances, but, like hacker,
-cyberpunk too has become a criminal pejorative today.)
-
-In any case, breaking into computer systems was hardly alien
-to the original hacker tradition. The first tottering systems
-of the 1960s required fairly extensive internal surgery merely
-to function day-by-day. Their users "invaded" the deepest,
-most arcane recesses of their operating software almost
-as a matter of routine. "Computer security" in these early,
-primitive systems was at best an afterthought. What security
-there was, was entirely physical, for it was assumed that
-anyone allowed near this expensive, arcane hardware would be
-a fully qualified professional expert.
-
-In a campus environment, though, this meant that grad students,
-teaching assistants, undergraduates, and eventually,
-all manner of dropouts and hangers-on ended up accessing
-and often running the works.
-
-Universities, even modern universities, are not in
-the business of maintaining security over information.
-On the contrary, universities, as institutions, pre-date
-the "information economy" by many centuries and are not-
-for-profit cultural entities, whose reason for existence
-(purportedly) is to discover truth, codify it through
-techniques of scholarship, and then teach it. Universities
-are meant to PASS THE TORCH OF CIVILIZATION, not just
-download data into student skulls, and the values of the
-academic community are strongly at odds with those of all
-would-be information empires. Teachers at all levels, from
-kindergarten up, have proven to be shameless and persistent
-software and data pirates. Universities do not merely
-"leak information" but vigorously broadcast free thought.
-
-This clash of values has been fraught with controversy.
-Many hackers of the 1960s remember their professional
-apprenticeship as a long guerilla war against the uptight
-mainframe-computer "information priesthood." These computer-hungry
-youngsters had to struggle hard for access to computing power,
-and many of them were not above certain, er, shortcuts.
-But, over the years, this practice freed computing
-from the sterile reserve of lab-coated technocrats and
-was largely responsible for the explosive growth of computing
-in general society--especially PERSONAL computing.
-
-Access to technical power acted like catnip on certain
-of these youngsters. Most of the basic techniques of
-computer intrusion: password cracking, trapdoors, backdoors,
-trojan horses--were invented in college environments in the 1960s,
-in the early days of network computing. Some off-the-cuff
-experience at computer intrusion was to be in the informal
-resume of most "hackers" and many future industry giants.
-Outside of the tiny cult of computer enthusiasts, few people
-thought much about the implications of "breaking into"
-computers. This sort of activity had not yet been publicized,
-much less criminalized.
-
-In the 1960s, definitions of "property" and "privacy"
-had not yet been extended to cyberspace. Computers
-were not yet indispensable to society. There were no vast
-databanks of vulnerable, proprietary information stored
-in computers, which might be accessed, copied without
-permission, erased, altered, or sabotaged. The stakes
-were low in the early days--but they grew every year,
-exponentially, as computers themselves grew.
-
-By the 1990s, commercial and political pressures
-had become overwhelming, and they broke the social
-boundaries of the hacking subculture. Hacking
-had become too important to be left to the hackers.
-Society was now forced to tackle the intangible nature
-of cyberspace-as-property, cyberspace as privately-owned
-unreal-estate. In the new, severe, responsible, high-stakes
-context of the "Information Society" of the 1990s,
-"hacking" was called into question.
-
-What did it mean to break into a computer without
-permission and use its computational power, or look
-around inside its files without hurting anything?
-What were computer-intruding hackers, anyway--how should
-society, and the law, best define their actions?
-Were they just BROWSERS, harmless intellectual explorers?
-Were they VOYEURS, snoops, invaders of privacy? Should
-they be sternly treated as potential AGENTS OF ESPIONAGE,
-or perhaps as INDUSTRIAL SPIES? Or were they best
-defined as TRESPASSERS, a very common teenage
-misdemeanor? Was hacking THEFT OF SERVICE?
-(After all, intruders were getting someone else's
-computer to carry out their orders, without permission
-and without paying). Was hacking FRAUD? Maybe it was
-best described as IMPERSONATION. The commonest mode
-of computer intrusion was (and is) to swipe or snoop
-somebody else's password, and then enter the computer
-in the guise of another person--who is commonly stuck
-with the blame and the bills.
-
-Perhaps a medical metaphor was better--hackers should
-be defined as "sick," as COMPUTER ADDICTS unable
-to control their irresponsible, compulsive behavior.
-
-But these weighty assessments meant little to the
-people who were actually being judged. From inside
-the underground world of hacking itself, all these
-perceptions seem quaint, wrongheaded, stupid, or meaningless.
-The most important self-perception of underground hackers--
-from the 1960s, right through to the present day--is that
-they are an ELITE. The day-to-day struggle in the underground
-is not over sociological definitions--who cares?--but for power,
-knowledge, and status among one's peers.
-
-When you are a hacker, it is your own inner conviction
-of your elite status that enables you to break, or let
-us say "transcend," the rules. It is not that ALL rules
-go by the board. The rules habitually broken by hackers
-are UNIMPORTANT rules--the rules of dopey greedhead telco
-bureaucrats and pig-ignorant government pests.
-
-Hackers have their OWN rules, which separate behavior
-which is cool and elite, from behavior which is rodentlike,
-stupid and losing. These "rules," however, are mostly unwritten
-and enforced by peer pressure and tribal feeling. Like all rules
-that depend on the unspoken conviction that everybody else
-is a good old boy, these rules are ripe for abuse. The mechanisms
-of hacker peer- pressure, "teletrials" and ostracism, are rarely used
-and rarely work. Back-stabbing slander, threats, and electronic
-harassment are also freely employed in down-and-dirty intrahacker feuds,
-but this rarely forces a rival out of the scene entirely. The only real
-solution for the problem of an utterly losing, treacherous and rodentlike
-hacker is to TURN HIM IN TO THE POLICE. Unlike the Mafia or Medellin Cartel,
-the hacker elite cannot simply execute the bigmouths, creeps and troublemakers
-among their ranks, so they turn one another in with astonishing frequency.
-
-There is no tradition of silence or OMERTA in the hacker underworld.
-Hackers can be shy, even reclusive, but when they do talk, hackers
-tend to brag, boast and strut. Almost everything hackers do is INVISIBLE;
-if they don't brag, boast, and strut about it, then NOBODY WILL EVER KNOW.
-If you don't have something to brag, boast, and strut about, then nobody
-in the underground will recognize you and favor you with vital cooperation
-and respect.
-
-The way to win a solid reputation in the underground
-is by telling other hackers things that could only
-have been learned by exceptional cunning and stealth.
-Forbidden knowledge, therefore, is the basic currency
-of the digital underground, like seashells among
-Trobriand Islanders. Hackers hoard this knowledge,
-and dwell upon it obsessively, and refine it,
-and bargain with it, and talk and talk about it.
-
-Many hackers even suffer from a strange obsession to TEACH--
-to spread the ethos and the knowledge of the digital underground.
-They'll do this even when it gains them no particular advantage
-and presents a grave personal risk.
-
-And when that risk catches up with them, they will go right on teaching
-and preaching--to a new audience this time, their interrogators from law
-enforcement. Almost every hacker arrested tells everything he knows--
-all about his friends, his mentors, his disciples--legends, threats,
-horror stories, dire rumors, gossip, hallucinations. This is, of course,
-convenient for law enforcement--except when law enforcement begins
-to believe hacker legendry.
-
-Phone phreaks are unique among criminals in their willingness
-to call up law enforcement officials--in the office, at their homes--
-and give them an extended piece of their mind. It is hard not to
-interpret this as BEGGING FOR ARREST, and in fact it is an act
-of incredible foolhardiness. Police are naturally nettled
-by these acts of chutzpah and will go well out of their way
-to bust these flaunting idiots. But it can also be interpreted
-as a product of a world-view so elitist, so closed and hermetic,
-that electronic police are simply not perceived as "police,"
-but rather as ENEMY PHONE PHREAKS who should be scolded
-into behaving "decently."
-
-Hackers at their most grandiloquent perceive themselves
-as the elite pioneers of a new electronic world.
-Attempts to make them obey the democratically
-established laws of contemporary American society are
-seen as repression and persecution. After all, they argue,
-if Alexander Graham Bell had gone along with the rules
-of the Western Union telegraph company, there would have
-been no telephones. If Jobs and Wozniak had believed
-that IBM was the be-all and end-all, there would have
-been no personal computers. If Benjamin Franklin and
-Thomas Jefferson had tried to "work within the system"
-there would have been no United States.
-
-Not only do hackers privately believe this as an article of faith,
-but they have been known to write ardent manifestos about it.
-Here are some revealing excerpts from an especially vivid hacker manifesto:
-"The Techno-Revolution" by "Dr. Crash," which appeared in electronic
-form in Phrack Volume 1, Issue 6, Phile 3.
-
-
-"To fully explain the true motives behind hacking,
-we must first take a quick look into the past. In the 1960s,
-a group of MIT students built the first modern computer system.
-This wild, rebellious group of young men were the first to bear
-the name `hackers.' The systems that they developed were intended
-to be used to solve world problems and to benefit all of mankind.
-"As we can see, this has not been the case. The computer system
-has been solely in the hands of big businesses and the government.
-The wonderful device meant to enrich life has become a weapon which
-dehumanizes people. To the government and large businesses,
-people are no more than disk space, and the government doesn't
-use computers to arrange aid for the poor, but to control nuclear
-death weapons. The average American can only have access
-to a small microcomputer which is worth only a fraction
-of what they pay for it. The businesses keep the
-true state-of-the-art equipment away from the people
-behind a steel wall of incredibly high prices and bureaucracy.
-It is because of this state of affairs that hacking was born. (. . .)
-"Of course, the government doesn't want the monopoly of technology broken,
-so they have outlawed hacking and arrest anyone who is caught. (. . .)
-The phone company is another example of technology abused and kept
-from people with high prices. (. . .) "Hackers often find that their
-existing equipment, due to the monopoly tactics of computer companies,
-is inefficient for their purposes. Due to the exorbitantly high prices,
-it is impossible to legally purchase the necessary equipment.
-This need has given still another segment of the fight: Credit Carding.
-Carding is a way of obtaining the necessary goods without paying for them.
-It is again due to the companies' stupidity that Carding is so easy,
-and shows that the world's businesses are in the hands of those
-with considerably less technical know-how than we, the hackers. (. . .)
-"Hacking must continue. We must train newcomers to the art of hacking.
-(. . . .) And whatever you do, continue the fight. Whether you know it
-or not, if you are a hacker, you are a revolutionary. Don't worry,
-you're on the right side."
-
-The defense of "carding" is rare. Most hackers regard credit-card
-theft as "poison" to the underground, a sleazy and immoral effort that,
-worse yet, is hard to get away with. Nevertheless, manifestos advocating
-credit-card theft, the deliberate crashing of computer systems,
-and even acts of violent physical destruction such as vandalism
-and arson do exist in the underground. These boasts and threats
-are taken quite seriously by the police. And not every hacker
-is an abstract, Platonic computer-nerd. Some few are quite experienced
-at picking locks, robbing phone-trucks, and breaking and entering buildings.
-
-Hackers vary in their degree of hatred for authority
-and the violence of their rhetoric. But, at a bottom line,
-they are scofflaws. They don't regard the current rules
-of electronic behavior as respectable efforts to preserve
-law and order and protect public safety. They regard these
-laws as immoral efforts by soulless corporations to protect
-their profit margins and to crush dissidents. "Stupid" people,
-including police, businessmen, politicians, and journalists,
-simply have no right to judge the actions of those possessed of genius,
-techno-revolutionary intentions, and technical expertise.
-
-#
-
-Hackers are generally teenagers and college kids not
-engaged in earning a living. They often come from fairly
-well-to-do middle-class backgrounds, and are markedly
-anti-materialistic (except, that is, when it comes to
-computer equipment). Anyone motivated by greed for
-mere money (as opposed to the greed for power,
-knowledge and status) is swiftly written-off as a narrow-
-minded breadhead whose interests can only be corrupt
-and contemptible. Having grown up in the 1970s and
-1980s, the young Bohemians of the digital underground
-regard straight society as awash in plutocratic corruption,
-where everyone from the President down is for sale and
-whoever has the gold makes the rules.
-
-Interestingly, there's a funhouse-mirror image of this attitude
-on the other side of the conflict. The police are also
-one of the most markedly anti-materialistic groups
-in American society, motivated not by mere money
-but by ideals of service, justice, esprit-de-corps,
-and, of course, their own brand of specialized knowledge
-and power. Remarkably, the propaganda war between cops
-and hackers has always involved angry allegations
-that the other side is trying to make a sleazy buck.
-Hackers consistently sneer that anti-phreak prosecutors
-are angling for cushy jobs as telco lawyers and that
-computer-crime police are aiming to cash in later
-as well-paid computer-security consultants in the private sector.
-
-For their part, police publicly conflate all
-hacking crimes with robbing payphones with crowbars.
-Allegations of "monetary losses" from computer intrusion
-are notoriously inflated. The act of illicitly copying
-a document from a computer is morally equated with
-directly robbing a company of, say, half a million dollars.
-The teenage computer intruder in possession of this "proprietary"
-document has certainly not sold it for such a sum, would likely
-have little idea how to sell it at all, and quite probably
-doesn't even understand what he has. He has not made a cent
-in profit from his felony but is still morally equated with
-a thief who has robbed the church poorbox and lit out for Brazil.
-
-Police want to believe that all hackers are thieves.
-It is a tortuous and almost unbearable act for the American
-justice system to put people in jail because they want
-to learn things which are forbidden for them to know.
-In an American context, almost any pretext for punishment
-is better than jailing people to protect certain restricted
-kinds of information. Nevertheless, POLICING INFORMATION
-is part and parcel of the struggle against hackers.
-
-This dilemma is well exemplified by the remarkable
-activities of "Emmanuel Goldstein," editor and publisher
-of a print magazine known as 2600: The Hacker Quarterly.
-Goldstein was an English major at Long Island's State University
-of New York in the '70s, when he became involved with the local
-college radio station. His growing interest in electronics
-caused him to drift into Yippie TAP circles and thus into
-the digital underground, where he became a self-described
-techno-rat. His magazine publishes techniques of computer
-intrusion and telephone "exploration" as well as gloating
-exposes of telco misdeeds and governmental failings.
-
-Goldstein lives quietly and very privately in a large,
-crumbling Victorian mansion in Setauket, New York.
-The seaside house is decorated with telco decals, chunks of
-driftwood, and the basic bric-a-brac of a hippie crash-pad.
-He is unmarried, mildly unkempt, and survives mostly
-on TV dinners and turkey-stuffing eaten straight out
-of the bag. Goldstein is a man of considerable charm
-and fluency, with a brief, disarming smile and the kind
-of pitiless, stubborn, thoroughly recidivist integrity
-that America's electronic police find genuinely alarming.
-
-Goldstein took his nom-de-plume, or "handle," from
-a character in Orwell's 1984, which may be taken,
-correctly, as a symptom of the gravity of his sociopolitical
-worldview. He is not himself a practicing computer
-intruder, though he vigorously abets these actions,
-especially when they are pursued against large
-corporations or governmental agencies. Nor is he a thief,
-for he loudly scorns mere theft of phone service, in favor
-of "exploring and manipulating the system." He is probably
-best described and understood as a DISSIDENT.
-
-Weirdly, Goldstein is living in modern America
-under conditions very similar to those of former
-East European intellectual dissidents. In other words,
-he flagrantly espouses a value-system that is deeply
-and irrevocably opposed to the system of those in power
-and the police. The values in 2600 are generally expressed
-in terms that are ironic, sarcastic, paradoxical, or just
-downright confused. But there's no mistaking their
-radically anti-authoritarian tenor. 2600 holds that
-technical power and specialized knowledge, of any kind
-obtainable, belong by right in the hands of those individuals
-brave and bold enough to discover them--by whatever means necessary.
-Devices, laws, or systems that forbid access, and the free
-spread of knowledge, are provocations that any free
-and self-respecting hacker should relentlessly attack.
-The "privacy" of governments, corporations and other soulless
-technocratic organizations should never be protected
-at the expense of the liberty and free initiative
-of the individual techno-rat.
-
-However, in our contemporary workaday world, both governments
-and corporations are very anxious indeed to police information
-which is secret, proprietary, restricted, confidential,
-copyrighted, patented, hazardous, illegal, unethical,
-embarrassing, or otherwise sensitive. This makes Goldstein
-persona non grata, and his philosophy a threat.
-
-Very little about the conditions of Goldstein's daily
-life would astonish, say, Vaclav Havel. (We may note
-in passing that President Havel once had his word-processor
-confiscated by the Czechoslovak police.) Goldstein lives
-by SAMIZDAT, acting semi-openly as a data-center
-for the underground, while challenging the powers-that-be
-to abide by their own stated rules: freedom of speech
-and the First Amendment.
-
-Goldstein thoroughly looks and acts the part of techno-rat,
-with shoulder-length ringlets and a piratical black
-fisherman's-cap set at a rakish angle. He often shows up
-like Banquo's ghost at meetings of computer professionals,
-where he listens quietly, half-smiling and taking thorough notes.
-
-Computer professionals generally meet publicly,
-and find it very difficult to rid themselves of Goldstein
-and his ilk without extralegal and unconstitutional actions.
-Sympathizers, many of them quite respectable people
-with responsible jobs, admire Goldstein's attitude and
-surreptitiously pass him information. An unknown but
-presumably large proportion of Goldstein's 2,000-plus
-readership are telco security personnel and police,
-who are forced to subscribe to 2600 to stay abreast
-of new developments in hacking. They thus find themselves
-PAYING THIS GUY'S RENT while grinding their teeth in anguish,
-a situation that would have delighted Abbie Hoffman
-(one of Goldstein's few idols).
-
-Goldstein is probably the best-known public representative
-of the hacker underground today, and certainly the best-hated.
-Police regard him as a Fagin, a corrupter of youth, and speak
-of him with untempered loathing. He is quite an accomplished gadfly.
-After the Martin Luther King Day Crash of 1990, Goldstein,
-for instance, adeptly rubbed salt into the wound in the pages of 2600.
-"Yeah, it was fun for the phone phreaks as we watched the network crumble,"
-he admitted cheerfully. "But it was also an ominous sign of what's
-to come. . . . Some AT&T people, aided by well-meaning but ignorant media,
-were spreading the notion that many companies had the same software
-and therefore could face the same problem someday. Wrong. This was
-entirely an AT&T software deficiency. Of course, other companies could
-face entirely DIFFERENT software problems. But then, so too could AT&T."
-
-After a technical discussion of the system's failings,
-the Long Island techno-rat went on to offer thoughtful
-criticism to the gigantic multinational's hundreds of
-professionally qualified engineers. "What we don't know
-is how a major force in communications like AT&T could
-be so sloppy. What happened to backups? Sure,
-computer systems go down all the time, but people
-making phone calls are not the same as people logging
-on to computers. We must make that distinction. It's not
-acceptable for the phone system or any other essential
-service to `go down.' If we continue to trust technology
-without understanding it, we can look forward to many
-variations on this theme.
-
-"AT&T owes it to its customers to be prepared to INSTANTLY
-switch to another network if something strange and unpredictable
-starts occurring. The news here isn't so much the failure
-of a computer program, but the failure of AT&T's entire structure."
-
-The very idea of this. . . . this PERSON. . . . offering
-"advice" about "AT&T's entire structure" is more than
-some people can easily bear. How dare this near-criminal
-dictate what is or isn't "acceptable" behavior from AT&T?
-Especially when he's publishing, in the very same issue,
-detailed schematic diagrams for creating various switching-network
-signalling tones unavailable to the public.
-
-"See what happens when you drop a `silver box' tone or two
-down your local exchange or through different long distance
-service carriers," advises 2600 contributor "Mr. Upsetter"
-in "How To Build a Signal Box." "If you experiment systematically
-and keep good records, you will surely discover something interesting."
-
-This is, of course, the scientific method, generally regarded
-as a praiseworthy activity and one of the flowers of modern civilization.
-One can indeed learn a great deal with this sort of structured
-intellectual activity. Telco employees regard this mode of "exploration"
-as akin to flinging sticks of dynamite into their pond to see what lives
-on the bottom.
-
-2600 has been published consistently since 1984.
-It has also run a bulletin board computer system,
-printed 2600 T-shirts, taken fax calls. . . .
-The Spring 1991 issue has an interesting announcement on page 45:
-"We just discovered an extra set of wires attached to our fax line
-and heading up the pole. (They've since been clipped.)
-Your faxes to us and to anyone else could be monitored."
-In the worldview of 2600, the tiny band of techno-rat brothers
-(rarely, sisters) are a beseiged vanguard of the truly free and honest.
-The rest of the world is a maelstrom of corporate crime and high-level
-governmental corruption, occasionally tempered with well-meaning
-ignorance. To read a few issues in a row is to enter a nightmare
-akin to Solzhenitsyn's, somewhat tempered by the fact that 2600
-is often extremely funny.
-
-Goldstein did not become a target of the Hacker Crackdown,
-though he protested loudly, eloquently, and publicly about it,
-and it added considerably to his fame. It was not that he is not
-regarded as dangerous, because he is so regarded. Goldstein has had
-brushes with the law in the past: in 1985, a 2600 bulletin board
-computer was seized by the FBI, and some software on it was formally
-declared "a burglary tool in the form of a computer program."
-But Goldstein escaped direct repression in 1990, because his
-magazine is printed on paper, and recognized as subject
-to Constitutional freedom of the press protection.
-As was seen in the Ramparts case, this is far from
-an absolute guarantee. Still, as a practical matter,
-shutting down 2600 by court-order would create so much
-legal hassle that it is simply unfeasible, at least
-for the present. Throughout 1990, both Goldstein
-and his magazine were peevishly thriving.
-
-Instead, the Crackdown of 1990 would concern itself
-with the computerized version of forbidden data.
-The crackdown itself, first and foremost, was about
-BULLETIN BOARD SYSTEMS. Bulletin Board Systems, most often
-known by the ugly and un-pluralizable acronym "BBS," are
-the life-blood of the digital underground. Boards were
-also central to law enforcement's tactics and strategy
-in the Hacker Crackdown.
-
-A "bulletin board system" can be formally defined as
-a computer which serves as an information and message-
-passing center for users dialing-up over the phone-lines
-through the use of modems. A "modem," or modulator-
-demodulator, is a device which translates the digital
-impulses of computers into audible analog telephone
-signals, and vice versa. Modems connect computers
-to phones and thus to each other.
-
-Large-scale mainframe computers have been connected since the 1960s,
-but PERSONAL computers, run by individuals out of their homes,
-were first networked in the late 1970s. The "board" created
-by Ward Christensen and Randy Suess in February 1978,
-in Chicago, Illinois, is generally regarded as the first
-personal-computer bulletin board system worthy of the name.
-
-Boards run on many different machines, employing many
-different kinds of software. Early boards were crude and buggy,
-and their managers, known as "system operators" or "sysops,"
-were hard-working technical experts who wrote their own software.
-But like most everything else in the world of electronics,
-boards became faster, cheaper, better-designed, and generally
-far more sophisticated throughout the 1980s. They also moved
-swiftly out of the hands of pioneers and into those of the
-general public. By 1985 there were something in the
-neighborhood of 4,000 boards in America. By 1990 it was
-calculated, vaguely, that there were about 30,000 boards in
-the US, with uncounted thousands overseas.
-
-Computer bulletin boards are unregulated enterprises.
-Running a board is a rough-and-ready, catch-as-catch-can proposition.
-Basically, anybody with a computer, modem, software and a phone-line
-can start a board. With second-hand equipment and public-domain
-free software, the price of a board might be quite small--
-less than it would take to publish a magazine or even a
-decent pamphlet. Entrepreneurs eagerly sell bulletin-board
-software, and will coach nontechnical amateur sysops in its use.
-
-Boards are not "presses." They are not magazines,
-or libraries, or phones, or CB radios, or traditional cork
-bulletin boards down at the local laundry, though they
-have some passing resemblance to those earlier media.
-Boards are a new medium--they may even be a LARGE NUMBER of new media.
-
-Consider these unique characteristics: boards are cheap,
-yet they can have a national, even global reach.
-Boards can be contacted from anywhere in the global
-telephone network, at NO COST to the person running the board--
-the caller pays the phone bill, and if the caller is local,
-the call is free. Boards do not involve an editorial elite
-addressing a mass audience. The "sysop" of a board is not
-an exclusive publisher or writer--he is managing an electronic salon,
-where individuals can address the general public, play the part
-of the general public, and also exchange private mail
-with other individuals. And the "conversation" on boards,
-though fluid, rapid, and highly interactive, is not spoken,
-but written. It is also relatively anonymous, sometimes completely so.
-
-And because boards are cheap and ubiquitous, regulations
-and licensing requirements would likely be practically unenforceable.
-It would almost be easier to "regulate," "inspect," and "license"
-the content of private mail--probably more so, since the mail system
-is operated by the federal government. Boards are run by individuals,
-independently, entirely at their own whim.
-
-For the sysop, the cost of operation is not the primary
-limiting factor. Once the investment in a computer and
-modem has been made, the only steady cost is the charge
-for maintaining a phone line (or several phone lines).
-The primary limits for sysops are time and energy.
-Boards require upkeep. New users are generally "validated"--
-they must be issued individual passwords, and called at
-home by voice-phone, so that their identity can be
-verified. Obnoxious users, who exist in plenty, must be
-chided or purged. Proliferating messages must be deleted
-when they grow old, so that the capacity of the system
-is not overwhelmed. And software programs (if such things
-are kept on the board) must be examined for possible
-computer viruses. If there is a financial charge to use
-the board (increasingly common, especially in larger and
-fancier systems) then accounts must be kept, and users
-must be billed. And if the board crashes--a very common
-occurrence--then repairs must be made.
-
-Boards can be distinguished by the amount of effort
-spent in regulating them. First, we have the completely
-open board, whose sysop is off chugging brews and
-watching re-runs while his users generally degenerate
-over time into peevish anarchy and eventual silence.
-Second comes the supervised board, where the sysop
-breaks in every once in a while to tidy up, calm brawls,
-issue announcements, and rid the community of dolts
-and troublemakers. Third is the heavily supervised
-board, which sternly urges adult and responsible behavior
-and swiftly edits any message considered offensive,
-impertinent, illegal or irrelevant. And last comes
-the completely edited "electronic publication," which
-is presented to a silent audience which is not allowed
-to respond directly in any way.
-
-Boards can also be grouped by their degree of anonymity.
-There is the completely anonymous board, where everyone
-uses pseudonyms--"handles"--and even the sysop is unaware
-of the user's true identity. The sysop himself is likely
-pseudonymous on a board of this type. Second, and rather
-more common, is the board where the sysop knows (or thinks
-he knows) the true names and addresses of all users,
-but the users don't know one another's names and may not know his.
-Third is the board where everyone has to use real names,
-and roleplaying and pseudonymous posturing are forbidden.
-
-Boards can be grouped by their immediacy. "Chat-lines"
-are boards linking several users together over several
-different phone-lines simultaneously, so that people
-exchange messages at the very moment that they type.
-(Many large boards feature "chat" capabilities along
-with other services.) Less immediate boards,
-perhaps with a single phoneline, store messages serially,
-one at a time. And some boards are only open for business
-in daylight hours or on weekends, which greatly slows response.
-A NETWORK of boards, such as "FidoNet," can carry electronic mail
-from board to board, continent to continent, across huge distances--
-but at a relative snail's pace, so that a message can take several
-days to reach its target audience and elicit a reply.
-
-Boards can be grouped by their degree of community.
-Some boards emphasize the exchange of private,
-person-to-person electronic mail. Others emphasize
-public postings and may even purge people who "lurk,"
-merely reading posts but refusing to openly participate.
-Some boards are intimate and neighborly. Others are frosty
-and highly technical. Some are little more than storage
-dumps for software, where users "download" and "upload" programs,
-but interact among themselves little if at all.
-
-Boards can be grouped by their ease of access. Some boards
-are entirely public. Others are private and restricted only
-to personal friends of the sysop. Some boards divide users by status.
-On these boards, some users, especially beginners, strangers or children,
-will be restricted to general topics, and perhaps forbidden to post.
-Favored users, though, are granted the ability to post as they please,
-and to stay "on-line" as long as they like, even to the disadvantage
-of other people trying to call in. High-status users can be given access
-to hidden areas in the board, such as off-color topics, private discussions,
-and/or valuable software. Favored users may even become "remote sysops"
-with the power to take remote control of the board through their own
-home computers. Quite often "remote sysops" end up doing all the work
-and taking formal control of the enterprise, despite the fact that it's
-physically located in someone else's house. Sometimes several "co-sysops"
-share power.
-
-And boards can also be grouped by size. Massive, nationwide
-commercial networks, such as CompuServe, Delphi, GEnie and Prodigy,
-are run on mainframe computers and are generally not considered "boards,"
-though they share many of their characteristics, such as electronic mail,
-discussion topics, libraries of software, and persistent and growing problems
-with civil-liberties issues. Some private boards have as many as
-thirty phone-lines and quite sophisticated hardware. And then
-there are tiny boards.
-
-Boards vary in popularity. Some boards are huge and crowded,
-where users must claw their way in against a constant busy-signal.
-Others are huge and empty--there are few things sadder than a formerly
-flourishing board where no one posts any longer, and the dead conversations
-of vanished users lie about gathering digital dust. Some boards are tiny
-and intimate, their telephone numbers intentionally kept confidential
-so that only a small number can log on.
-
-And some boards are UNDERGROUND.
-
-Boards can be mysterious entities. The activities of
-their users can be hard to differentiate from conspiracy.
-Sometimes they ARE conspiracies. Boards have harbored,
-or have been accused of harboring, all manner of fringe groups,
-and have abetted, or been accused of abetting, every manner
-of frowned-upon, sleazy, radical, and criminal activity.
-There are Satanist boards. Nazi boards. Pornographic boards.
-Pedophile boards. Drug- dealing boards. Anarchist boards.
-Communist boards. Gay and Lesbian boards (these exist in great profusion,
-many of them quite lively with well-established histories).
-Religious cult boards. Evangelical boards. Witchcraft
-boards, hippie boards, punk boards, skateboarder boards.
-Boards for UFO believers. There may well be boards for
-serial killers, airline terrorists and professional assassins.
-There is simply no way to tell. Boards spring up, flourish,
-and disappear in large numbers, in most every corner of
-the developed world. Even apparently innocuous public
-boards can, and sometimes do, harbor secret areas known
-only to a few. And even on the vast, public, commercial services,
-private mail is very private--and quite possibly criminal.
-
-Boards cover most every topic imaginable and some
-that are hard to imagine. They cover a vast spectrum
-of social activity. However, all board users do have
-something in common: their possession of computers
-and phones. Naturally, computers and phones are
-primary topics of conversation on almost every board.
-
-And hackers and phone phreaks, those utter devotees
-of computers and phones, live by boards. They swarm by boards.
-They are bred by boards. By the late 1980s, phone-phreak groups
-and hacker groups, united by boards, had proliferated fantastically.
-
-
-As evidence, here is a list of hacker groups compiled
-by the editors of Phrack on August 8, 1988.
-
-
-The Administration.
-Advanced Telecommunications, Inc.
-ALIAS.
-American Tone Travelers.
-Anarchy Inc.
-Apple Mafia.
-The Association.
-Atlantic Pirates Guild.
-
-Bad Ass Mother Fuckers.
-Bellcore.
-Bell Shock Force.
-Black Bag.
-
-Camorra.
-C&M Productions.
-Catholics Anonymous.
-Chaos Computer Club.
-Chief Executive Officers.
-Circle Of Death.
-Circle Of Deneb.
-Club X.
-Coalition of Hi-Tech
-Pirates.
-Coast-To-Coast.
-Corrupt Computing.
-Cult Of The
-Dead Cow.
-Custom Retaliations.
-
-Damage Inc.
-D&B Communications.
-The Danger Gang.
-Dec Hunters.
-Digital Gang.
-DPAK.
-
-Eastern Alliance.
-The Elite Hackers Guild.
-Elite Phreakers and Hackers Club.
-The Elite Society Of America.
-EPG.
-Executives Of Crime.
-Extasyy Elite.
-
-Fargo 4A.
-Farmers Of Doom.
-The Federation.
-Feds R Us.
-First Class.
-Five O.
-Five Star.
-Force Hackers.
-The 414s.
-
-Hack-A-Trip.
-Hackers Of America.
-High Mountain Hackers.
-High Society.
-The Hitchhikers.
-
-IBM Syndicate.
-The Ice Pirates.
-Imperial Warlords.
-Inner Circle.
-Inner Circle II.
-Insanity Inc.
-International Computer Underground Bandits.
-
-Justice League of America.
-
-Kaos Inc.
-Knights Of Shadow.
-Knights Of The Round Table.
-
-League Of Adepts.
-Legion Of Doom.
-Legion Of Hackers.
-Lords Of Chaos.
-Lunatic Labs, Unlimited.
-
-Master Hackers.
-MAD!
-The Marauders.
-MD/PhD.
-
-Metal Communications, Inc.
-MetalliBashers, Inc.
-MBI.
-
-Metro Communications.
-Midwest Pirates Guild.
-
-NASA Elite.
-The NATO Association.
-Neon Knights.
-
-Nihilist Order.
-Order Of The Rose.
-OSS.
-
-Pacific Pirates Guild.
-Phantom Access Associates.
-
-PHido PHreaks.
-The Phirm.
-Phlash.
-PhoneLine Phantoms.
-Phone Phreakers Of America.
-Phortune 500.
-
-Phreak Hack Delinquents.
-Phreak Hack Destroyers.
-
-Phreakers, Hackers, And Laundromat Employees Gang (PHALSE Gang).
-Phreaks Against Geeks.
-Phreaks Against Phreaks Against Geeks.
-Phreaks and Hackers of America.
-Phreaks Anonymous World Wide.
-Project Genesis.
-The Punk Mafia.
-
-The Racketeers.
-Red Dawn Text Files.
-Roscoe Gang.
-
-
-SABRE.
-Secret Circle of Pirates.
-Secret Service.
-707 Club.
-Shadow Brotherhood.
-Sharp Inc.
-65C02 Elite.
-
-Spectral Force.
-Star League.
-Stowaways.
-Strata-Crackers.
-
-
-Team Hackers '86.
-Team Hackers '87.
-
-TeleComputist Newsletter Staff.
-Tribunal Of Knowledge.
-
-Triple Entente.
-Turn Over And Die Syndrome (TOADS).
-
-300 Club.
-1200 Club.
-2300 Club.
-2600 Club.
-2601 Club.
-
-2AF.
-
-The United Soft WareZ Force.
-United Technical Underground.
-
-Ware Brigade.
-The Warelords.
-WASP.
-
-Contemplating this list is an impressive, almost humbling business.
-As a cultural artifact, the thing approaches poetry.
-
-Underground groups--subcultures--can be distinguished
-from independent cultures by their habit of referring
-constantly to the parent society. Undergrounds by their
-nature constantly must maintain a membrane of differentiation.
-Funny/distinctive clothes and hair, specialized jargon, specialized
-ghettoized areas in cities, different hours of rising, working,
-sleeping. . . . The digital underground, which specializes in information,
-relies very heavily on language to distinguish itself. As can be seen
-from this list, they make heavy use of parody and mockery.
-It's revealing to see who they choose to mock.
-
-First, large corporations. We have the Phortune 500,
-The Chief Executive Officers, Bellcore, IBM Syndicate,
-SABRE (a computerized reservation service maintained
-by airlines). The common use of "Inc." is telling--
-none of these groups are actual corporations,
-but take clear delight in mimicking them.
-
-Second, governments and police. NASA Elite, NATO Association.
-"Feds R Us" and "Secret Service" are fine bits of fleering boldness.
-OSS--the Office of Strategic Services was the forerunner of the CIA.
-
-Third, criminals. Using stigmatizing pejoratives as a perverse
-badge of honor is a time-honored tactic for subcultures:
-punks, gangs, delinquents, mafias, pirates, bandits, racketeers.
-
-Specialized orthography, especially the use of "ph" for "f"
-and "z" for the plural "s," are instant recognition symbols.
-So is the use of the numeral "0" for the letter "O"
---computer-software orthography generally features a
-slash through the zero, making the distinction obvious.
-
-Some terms are poetically descriptive of computer intrusion:
-the Stowaways, the Hitchhikers, the PhoneLine Phantoms, Coast-to-Coast.
-Others are simple bravado and vainglorious puffery.
-(Note the insistent use of the terms "elite" and "master.")
-Some terms are blasphemous, some obscene, others merely cryptic--
-anything to puzzle, offend, confuse, and keep the straights at bay.
-
-Many hacker groups further re-encrypt their names
-by the use of acronyms: United Technical Underground
-becomes UTU, Farmers of Doom become FoD, the United SoftWareZ
-Force becomes, at its own insistence, "TuSwF," and woe to the
-ignorant rodent who capitalizes the wrong letters.
-
-It should be further recognized that the members of these groups
-are themselves pseudonymous. If you did, in fact, run across
-the "PhoneLine Phantoms," you would find them to consist of
-"Carrier Culprit," "The Executioner," "Black Majik,"
-"Egyptian Lover," "Solid State," and "Mr Icom."
-"Carrier Culprit" will likely be referred to by his friends
-as "CC," as in, "I got these dialups from CC of PLP."
-
-It's quite possible that this entire list refers to as
-few as a thousand people. It is not a complete list
-of underground groups--there has never been such a list,
-and there never will be. Groups rise, flourish, decline,
-share membership, maintain a cloud of wannabes and
-casual hangers-on. People pass in and out, are ostracized,
-get bored, are busted by police, or are cornered by telco
-security and presented with huge bills. Many "underground
-groups" are software pirates, "warez d00dz," who might break
-copy protection and pirate programs, but likely wouldn't dare
-to intrude on a computer-system.
-
-It is hard to estimate the true population of the digital
-underground. There is constant turnover. Most hackers
-start young, come and go, then drop out at age 22--
-the age of college graduation. And a large majority
-of "hackers" access pirate boards, adopt a handle,
-swipe software and perhaps abuse a phone-code or two,
-while never actually joining the elite.
-
-Some professional informants, who make it their business
-to retail knowledge of the underground to paymasters in private
-corporate security, have estimated the hacker population
-at as high as fifty thousand. This is likely highly inflated,
-unless one counts every single teenage software pirate
-and petty phone-booth thief. My best guess is about 5,000 people.
-Of these, I would guess that as few as a hundred are truly "elite"
---active computer intruders, skilled enough to penetrate
-sophisticated systems and truly to worry corporate security
-and law enforcement.
-
-Another interesting speculation is whether this group
-is growing or not. Young teenage hackers are often
-convinced that hackers exist in vast swarms and will soon
-dominate the cybernetic universe. Older and wiser
-veterans, perhaps as wizened as 24 or 25 years old,
-are convinced that the glory days are long gone, that the cops
-have the underground's number now, and that kids these days
-are dirt-stupid and just want to play Nintendo.
-
-My own assessment is that computer intrusion, as a non-profit act
-of intellectual exploration and mastery, is in slow decline,
-at least in the United States; but that electronic fraud,
-especially telecommunication crime, is growing by leaps and bounds.
-
-One might find a useful parallel to the digital underground
-in the drug underground. There was a time, now much-obscured
-by historical revisionism, when Bohemians freely shared joints
-at concerts, and hip, small-scale marijuana dealers might
-turn people on just for the sake of enjoying a long stoned conversation
-about the Doors and Allen Ginsberg. Now drugs are increasingly verboten,
-except in a high-stakes, highly-criminal world of highly addictive drugs.
-Over years of disenchantment and police harassment, a vaguely ideological,
-free-wheeling drug underground has relinquished the business of drug-dealing
-to a far more savage criminal hard-core. This is not a pleasant prospect
-to contemplate, but the analogy is fairly compelling.
-
-What does an underground board look like? What distinguishes
-it from a standard board? It isn't necessarily the conversation--
-hackers often talk about common board topics, such as hardware, software,
-sex, science fiction, current events, politics, movies, personal gossip.
-Underground boards can best be distinguished by their files, or "philes,"
-pre-composed texts which teach the techniques and ethos of the underground.
-These are prized reservoirs of forbidden knowledge. Some are anonymous,
-but most proudly bear the handle of the "hacker" who has created them,
-and his group affiliation, if he has one.
-
-Here is a partial table-of-contents of philes from an underground board,
-somewhere in the heart of middle America, circa 1991. The descriptions
-are mostly self-explanatory.
-
-
-BANKAMER.ZIP 5406 06-11-91 Hacking Bank America
-CHHACK.ZIP 4481 06-11-91 Chilton Hacking
-CITIBANK.ZIP 4118 06-11-91 Hacking Citibank
-CREDIMTC.ZIP 3241 06-11-91 Hacking Mtc Credit Company
-DIGEST.ZIP 5159 06-11-91 Hackers Digest
-HACK.ZIP 14031 06-11-91 How To Hack
-HACKBAS.ZIP 5073 06-11-91 Basics Of Hacking
-HACKDICT.ZIP 42774 06-11-91 Hackers Dictionary
-HACKER.ZIP 57938 06-11-91 Hacker Info
-HACKERME.ZIP 3148 06-11-91 Hackers Manual
-HACKHAND.ZIP 4814 06-11-91 Hackers Handbook
-HACKTHES.ZIP 48290 06-11-91 Hackers Thesis
-HACKVMS.ZIP 4696 06-11-91 Hacking Vms Systems
-MCDON.ZIP 3830 06-11-91 Hacking Macdonalds (Home Of The Archs)
-P500UNIX.ZIP 15525 06-11-91 Phortune 500 Guide To Unix
-RADHACK.ZIP 8411 06-11-91 Radio Hacking
-TAOTRASH.DOC 4096 12-25-89 Suggestions For Trashing
-TECHHACK.ZIP 5063 06-11-91 Technical Hacking
-
-
-The files above are do-it-yourself manuals about computer intrusion.
-The above is only a small section of a much larger library of hacking
-and phreaking techniques and history. We now move into a different
-and perhaps surprising area.
-
-+------------+
- |Anarchy|
-+------------+
-
-ANARC.ZIP 3641 06-11-91 Anarchy Files
-ANARCHST.ZIP 63703 06-11-91 Anarchist Book
-ANARCHY.ZIP 2076 06-11-91 Anarchy At Home
-ANARCHY3.ZIP 6982 06-11-91 Anarchy No 3
-ANARCTOY.ZIP 2361 06-11-91 Anarchy Toys
-ANTIMODM.ZIP 2877 06-11-91 Anti-modem Weapons
-ATOM.ZIP 4494 06-11-91 How To Make An Atom Bomb
-BARBITUA.ZIP 3982 06-11-91 Barbiturate Formula
-BLCKPWDR.ZIP 2810 06-11-91 Black Powder Formulas
-BOMB.ZIP 3765 06-11-91 How To Make Bombs
-BOOM.ZIP 2036 06-11-91 Things That Go Boom
-CHLORINE.ZIP 1926 06-11-91 Chlorine Bomb
-COOKBOOK.ZIP 1500 06-11-91 Anarchy Cook Book
-DESTROY.ZIP 3947 06-11-91 Destroy Stuff
-DUSTBOMB.ZIP 2576 06-11-91 Dust Bomb
-ELECTERR.ZIP 3230 06-11-91 Electronic Terror
-EXPLOS1.ZIP 2598 06-11-91 Explosives 1
-EXPLOSIV.ZIP 18051 06-11-91 More Explosives
-EZSTEAL.ZIP 4521 06-11-91 Ez-stealing
-FLAME.ZIP 2240 06-11-91 Flame Thrower
-FLASHLT.ZIP 2533 06-11-91 Flashlight Bomb
-FMBUG.ZIP 2906 06-11-91 How To Make An Fm Bug
-OMEEXPL.ZIP 2139 06-11-91 Home Explosives
-HOW2BRK.ZIP 3332 06-11-91 How To Break In
-LETTER.ZIP 2990 06-11-91 Letter Bomb
-LOCK.ZIP 2199 06-11-91 How To Pick Locks
-MRSHIN.ZIP 3991 06-11-91 Briefcase Locks
-NAPALM.ZIP 3563 06-11-91 Napalm At Home
-NITRO.ZIP 3158 06-11-91 Fun With Nitro
-PARAMIL.ZIP 2962 06-11-91 Paramilitary Info
-PICKING.ZIP 3398 06-11-91 Picking Locks
-PIPEBOMB.ZIP 2137 06-11-91 Pipe Bomb
-POTASS.ZIP 3987 06-11-91 Formulas With Potassium
-PRANK.TXT 11074 08-03-90 More Pranks To Pull On Idiots!
-REVENGE.ZIP 4447 06-11-91 Revenge Tactics
-ROCKET.ZIP 2590 06-11-91 Rockets For Fun
-SMUGGLE.ZIP 3385 06-11-91 How To Smuggle
-
-HOLY COW! The damned thing is full of stuff about bombs!
-
-What are we to make of this?
-
-First, it should be acknowledged that spreading
-knowledge about demolitions to teenagers is a highly and
-deliberately antisocial act. It is not, however, illegal.
-
-Second, it should be recognized that most of these
-philes were in fact WRITTEN by teenagers. Most adult
-American males who can remember their teenage years
-will recognize that the notion of building a flamethrower
-in your garage is an incredibly neat-o idea. ACTUALLY,
-building a flamethrower in your garage, however, is
-fraught with discouraging difficulty. Stuffing gunpowder
-into a booby-trapped flashlight, so as to blow the arm off
-your high-school vice-principal, can be a thing of dark
-beauty to contemplate. Actually committing assault by
-explosives will earn you the sustained attention of the
-federal Bureau of Alcohol, Tobacco and Firearms.
-
-Some people, however, will actually try these plans.
-A determinedly murderous American teenager can probably
-buy or steal a handgun far more easily than he can brew
-fake "napalm" in the kitchen sink. Nevertheless,
-if temptation is spread before people, a certain number
-will succumb, and a small minority will actually attempt
-these stunts. A large minority of that small minority
-will either fail or, quite likely, maim themselves,
-since these "philes" have not been checked for accuracy,
-are not the product of professional experience,
-and are often highly fanciful. But the gloating menace
-of these philes is not to be entirely dismissed.
-
-Hackers may not be "serious" about bombing; if they were,
-we would hear far more about exploding flashlights, homemade bazookas,
-and gym teachers poisoned by chlorine and potassium.
-However, hackers are VERY serious about forbidden knowledge.
-They are possessed not merely by curiosity, but by
-a positive LUST TO KNOW. The desire to know what
-others don't is scarcely new. But the INTENSITY
-of this desire, as manifested by these young technophilic
-denizens of the Information Age, may in fact BE new,
-and may represent some basic shift in social values--
-a harbinger of what the world may come to, as society
-lays more and more value on the possession,
-assimilation and retailing of INFORMATION
-as a basic commodity of daily life.
-
-There have always been young men with obsessive interests
-in these topics. Never before, however, have they been able
-to network so extensively and easily, and to propagandize
-their interests with impunity to random passers-by.
-High-school teachers will recognize that there's always
-one in a crowd, but when the one in a crowd escapes control
-by jumping into the phone-lines, and becomes a hundred such kids
-all together on a board, then trouble is brewing visibly.
-The urge of authority to DO SOMETHING, even something drastic,
-is hard to resist. And in 1990, authority did something.
-In fact authority did a great deal.
-
-#
-
-The process by which boards create hackers goes something
-like this. A youngster becomes interested in computers--
-usually, computer games. He hears from friends that
-"bulletin boards" exist where games can be obtained for free.
-(Many computer games are "freeware," not copyrighted--
-invented simply for the love of it and given away to the public;
-some of these games are quite good.) He bugs his parents for a modem,
-or quite often, uses his parents' modem.
-
-The world of boards suddenly opens up. Computer games
-can be quite expensive, real budget-breakers for a kid,
-but pirated games, stripped of copy protection, are cheap or free.
-They are also illegal, but it is very rare, almost unheard of,
-for a small-scale software pirate to be prosecuted.
-Once "cracked" of its copy protection, the program,
-being digital data, becomes infinitely reproducible.
-Even the instructions to the game, any manuals that accompany it,
-can be reproduced as text files, or photocopied from legitimate sets.
-Other users on boards can give many useful hints in game-playing tactics.
-And a youngster with an infinite supply of free computer games can
-certainly cut quite a swath among his modem-less friends.
-
-And boards are pseudonymous. No one need know that you're
-fourteen years old--with a little practice at subterfuge,
-you can talk to adults about adult things, and be accepted
-and taken seriously! You can even pretend to be a girl,
-or an old man, or anybody you can imagine. If you find this
-kind of deception gratifying, there is ample opportunity
-to hone your ability on boards.
-
-But local boards can grow stale. And almost every board maintains
-a list of phone-numbers to other boards, some in distant, tempting,
-exotic locales. Who knows what they're up to, in Oregon or Alaska
-or Florida or California? It's very easy to find out--just order
-the modem to call through its software--nothing to this, just typing
-on a keyboard, the same thing you would do for most any computer game.
-The machine reacts swiftly and in a few seconds you are talking to
-a bunch of interesting people on another seaboard.
-
-And yet the BILLS for this trivial action can be staggering!
-Just by going tippety-tap with your fingers, you may have
-saddled your parents with four hundred bucks in long-distance charges,
-and gotten chewed out but good. That hardly seems fair.
-
-How horrifying to have made friends in another state
-and to be deprived of their company--and their software--
-just because telephone companies demand absurd amounts of money!
-How painful, to be restricted to boards in one's own AREA CODE--
-what the heck is an "area code" anyway, and what makes it so special?
-A few grumbles, complaints, and innocent questions of this sort
-will often elicit a sympathetic reply from another board user--
-someone with some stolen codes to hand. You dither a while,
-knowing this isn't quite right, then you make up your mind
-to try them anyhow--AND THEY WORK! Suddenly you're doing something
-even your parents can't do. Six months ago you were just some kid--now,
-you're the Crimson Flash of Area Code 512! You're bad--you're nationwide!
-
-Maybe you'll stop at a few abused codes. Maybe you'll decide that
-boards aren't all that interesting after all, that it's wrong,
-not worth the risk --but maybe you won't. The next step
-is to pick up your own repeat-dialling program--
-to learn to generate your own stolen codes.
-(This was dead easy five years ago, much harder
-to get away with nowadays, but not yet impossible.)
-And these dialling programs are not complex or intimidating--
-some are as small as twenty lines of software.
-
-Now, you too can share codes. You can trade codes to learn
-other techniques. If you're smart enough to catch on,
-and obsessive enough to want to bother, and ruthless enough
-to start seriously bending rules, then you'll get better, fast.
-You start to develop a rep. You move up to a heavier class
-of board--a board with a bad attitude, the kind of board
-that naive dopes like your classmates and your former self
-have never even heard of! You pick up the jargon of phreaking
-and hacking from the board. You read a few of those anarchy philes--
-and man, you never realized you could be a real OUTLAW without
-ever leaving your bedroom.
-
-You still play other computer games, but now you have a new
-and bigger game. This one will bring you a different kind of status
-than destroying even eight zillion lousy space invaders.
-
-Hacking is perceived by hackers as a "game." This is
-not an entirely unreasonable or sociopathic perception.
-You can win or lose at hacking, succeed or fail,
-but it never feels "real." It's not simply that
-imaginative youngsters sometimes have a hard time
-telling "make-believe" from "real life." Cyberspace
-is NOT REAL! "Real" things are physical objects
-like trees and shoes and cars. Hacking takes place
-on a screen. Words aren't physical, numbers
-(even telephone numbers and credit card numbers)
-aren't physical. Sticks and stones may break my bones,
-but data will never hurt me. Computers SIMULATE reality,
-like computer games that simulate tank battles or dogfights
-or spaceships. Simulations are just make-believe,
-and the stuff in computers is NOT REAL.
-
-Consider this: if "hacking" is supposed to be so serious and
-real-life and dangerous, then how come NINE-YEAR-OLD KIDS have
-computers and modems? You wouldn't give a nine year old his own car,
-or his own rifle, or his own chainsaw--those things are "real."
-
-People underground are perfectly aware that the "game"
-is frowned upon by the powers that be. Word gets around
-about busts in the underground. Publicizing busts is one
-of the primary functions of pirate boards, but they also
-promulgate an attitude about them, and their own idiosyncratic
-ideas of justice. The users of underground boards won't complain
-if some guy is busted for crashing systems, spreading viruses,
-or stealing money by wire-fraud. They may shake their heads
-with a sneaky grin, but they won't openly defend these practices.
-But when a kid is charged with some theoretical amount of theft:
-$233,846.14, for instance, because he sneaked into a computer
-and copied something, and kept it in his house on a floppy disk--
-this is regarded as a sign of near-insanity from prosecutors,
-a sign that they've drastically mistaken the immaterial game
-of computing for their real and boring everyday world
-of fatcat corporate money.
-
-It's as if big companies and their suck-up lawyers
-think that computing belongs to them, and they can
-retail it with price stickers, as if it were boxes
-of laundry soap! But pricing "information" is like
-trying to price air or price dreams. Well, anybody
-on a pirate board knows that computing can be,
-and ought to be, FREE. Pirate boards are little
-independent worlds in cyberspace, and they don't belong
-to anybody but the underground. Underground boards
-aren't "brought to you by Procter & Gamble."
-
-To log on to an underground board can mean to
-experience liberation, to enter a world where,
-for once, money isn't everything and adults
-don't have all the answers.
-
-Let's sample another vivid hacker manifesto. Here are
-some excerpts from "The Conscience of a Hacker," by "The Mentor,"
-from Phrack Volume One, Issue 7, Phile 3.
-
-"I made a discovery today. I found a computer.
-Wait a second, this is cool. It does what I want it to.
-If it makes a mistake, it's because I screwed it up.
-Not because it doesn't like me. (. . .)
-"And then it happened. . .a door opened to a world. . .
-rushing through the phone line like heroin through an
-addict's veins, an electronic pulse is sent out,
-a refuge from day-to-day incompetencies is sought. . .
-a board is found. `This is it. . .this is where I belong. . .'
-"I know everyone here. . .even if I've never met them,
-never talked to them, may never hear from them again. . .
-I know you all. . . (. . .)
-
-"This is our world now. . .the world of the electron
-and the switch, the beauty of the baud. We make use of a
-service already existing without paying for what could be
-dirt-cheap if it wasn't run by profiteering gluttons, and you
-call us criminals. We explore. . .and you call us criminals.
-We seek after knowledge. . .and you call us criminals.
-We exist without skin color, without nationality,
-without religious bias. . .and you call us criminals.
-You build atomic bombs, you wage wars, you murder,
-cheat and lie to us and try to make us believe that
-it's for our own good, yet we're the criminals.
-
-"Yes, I am a criminal. My crime is that of curiosity.
-My crime is that of judging people by what they say and think,
-not what they look like. My crime is that of outsmarting you,
-something that you will never forgive me for."
-
-#
-
-There have been underground boards almost as long
-as there have been boards. One of the first was 8BBS,
-which became a stronghold of the West Coast phone-phreak elite.
-After going on-line in March 1980, 8BBS sponsored "Susan Thunder,"
-and "Tuc," and, most notoriously, "the Condor." "The Condor"
-bore the singular distinction of becoming the most vilified
-American phreak and hacker ever. Angry underground associates,
-fed up with Condor's peevish behavior, turned him in to police,
-along with a heaping double-helping of outrageous hacker legendry.
-As a result, Condor was kept in solitary confinement for seven months,
-for fear that he might start World War Three by triggering missile silos
-from the prison payphone. (Having served his time, Condor is now
-walking around loose; WWIII has thus far conspicuously failed to occur.)
-
-The sysop of 8BBS was an ardent free-speech enthusiast
-who simply felt that ANY attempt to restrict the expression
-of his users was unconstitutional and immoral.
-Swarms of the technically curious entered 8BBS
-and emerged as phreaks and hackers, until, in 1982,
-a friendly 8BBS alumnus passed the sysop a new modem
-which had been purchased by credit-card fraud.
-Police took this opportunity to seize the entire board
-and remove what they considered an attractive nuisance.
-
-Plovernet was a powerful East Coast pirate board
-that operated in both New York and Florida.
-Owned and operated by teenage hacker "Quasi Moto,"
-Plovernet attracted five hundred eager users in 1983.
-"Emmanuel Goldstein" was one-time co-sysop of Plovernet,
-along with "Lex Luthor," founder of the "Legion of Doom" group.
-Plovernet bore the signal honor of being the original home
-of the "Legion of Doom," about which the reader will be hearing
-a great deal, soon.
-
-"Pirate-80," or "P-80," run by a sysop known as "Scan-Man,"
-got into the game very early in Charleston, and continued
-steadily for years. P-80 flourished so flagrantly that
-even its most hardened users became nervous, and some
-slanderously speculated that "Scan Man" must have ties
-to corporate security, a charge he vigorously denied.
-
-"414 Private" was the home board for the first GROUP
-to attract conspicuous trouble, the teenage "414 Gang,"
-whose intrusions into Sloan-Kettering Cancer Center and
-Los Alamos military computers were to be a nine-days-wonder in 1982.
-
-At about this time, the first software piracy boards
-began to open up, trading cracked games for the Atari 800
-and the Commodore C64. Naturally these boards were
-heavily frequented by teenagers. And with the 1983
-release of the hacker-thriller movie War Games,
-the scene exploded. It seemed that every kid
-in America had demanded and gotten a modem for Christmas.
-Most of these dabbler wannabes put their modems in the attic
-after a few weeks, and most of the remainder minded their
-P's and Q's and stayed well out of hot water. But some
-stubborn and talented diehards had this hacker kid in
-War Games figured for a happening dude. They simply
-could not rest until they had contacted the underground--
-or, failing that, created their own.
-
-In the mid-80s, underground boards sprang up like digital fungi.
-ShadowSpawn Elite. Sherwood Forest I, II, and III.
-Digital Logic Data Service in Florida, sysoped by no less
-a man than "Digital Logic" himself; Lex Luthor of the
-Legion of Doom was prominent on this board, since it
-was in his area code. Lex's own board, "Legion of Doom,"
-started in 1984. The Neon Knights ran a network of Apple-
-hacker boards: Neon Knights North, South, East and West.
-Free World II was run by "Major Havoc." Lunatic Labs
-is still in operation as of this writing. Dr. Ripco
-in Chicago, an anything-goes anarchist board with an
-extensive and raucous history, was seized by Secret Service
-agents in 1990 on Sundevil day, but up again almost immediately,
-with new machines and scarcely diminished vigor.
-
-The St. Louis scene was not to rank with major centers
-of American hacking such as New York and L.A. But St.
-Louis did rejoice in possession of "Knight Lightning"
-and "Taran King," two of the foremost JOURNALISTS native
-to the underground. Missouri boards like Metal Shop,
-Metal Shop Private, Metal Shop Brewery, may not have
-been the heaviest boards around in terms of illicit
-expertise. But they became boards where hackers could
-exchange social gossip and try to figure out what the
-heck was going on nationally--and internationally.
-Gossip from Metal Shop was put into the form of news files,
-then assembled into a general electronic publication,
-Phrack, a portmanteau title coined from "phreak" and "hack."
-The Phrack editors were as obsessively curious about other
-hackers as hackers were about machines.
-
-Phrack, being free of charge and lively reading, began
-to circulate throughout the underground. As Taran King
-and Knight Lightning left high school for college,
-Phrack began to appear on mainframe machines linked to BITNET,
-and, through BITNET to the "Internet," that loose but
-extremely potent not-for-profit network where academic,
-governmental and corporate machines trade data through
-the UNIX TCP/IP protocol. (The "Internet Worm" of
-November 2-3,1988, created by Cornell grad student Robert Morris,
-was to be the largest and best-publicized computer-intrusion scandal
-to date. Morris claimed that his ingenious "worm" program was meant
-to harmlessly explore the Internet, but due to bad programming,
-the Worm replicated out of control and crashed some six thousand
-Internet computers. Smaller-scale and less ambitious Internet hacking
-was a standard for the underground elite.)
-
-Most any underground board not hopelessly lame and out-of-it
-would feature a complete run of Phrack--and, possibly,
-the lesser-known standards of the underground:
-the Legion of Doom Technical Journal, the obscene
-and raucous Cult of the Dead Cow files, P/HUN magazine,
-Pirate, the Syndicate Reports, and perhaps the highly
-anarcho-political Activist Times Incorporated.
-
-Possession of Phrack on one's board was prima facie
-evidence of a bad attitude. Phrack was seemingly everywhere,
-aiding, abetting, and spreading the underground ethos.
-And this did not escape the attention of corporate security
-or the police.
-
-We now come to the touchy subject of police and boards.
-Police, do, in fact, own boards. In 1989, there were
-police-sponsored boards in California, Colorado, Florida,
-Georgia, Idaho, Michigan, Missouri, Texas, and Virginia:
-boards such as "Crime Bytes," "Crimestoppers," "All Points"
-and "Bullet-N-Board." Police officers, as private computer
-enthusiasts, ran their own boards in Arizona, California,
-Colorado, Connecticut, Florida, Missouri, Maryland,
-New Mexico, North Carolina, Ohio, Tennessee and Texas.
-Police boards have often proved helpful in community relations.
-Sometimes crimes are reported on police boards.
-
-Sometimes crimes are COMMITTED on police boards.
-This has sometimes happened by accident, as naive hackers
-blunder onto police boards and blithely begin offering telephone codes.
-Far more often, however, it occurs through the now almost-traditional
-use of "sting boards." The first police sting-boards were established
-in 1985: "Underground Tunnel" in Austin, Texas, whose sysop
-Sgt. Robert Ansley called himself "Pluto"--"The Phone Company"
-in Phoenix, Arizona, run by Ken MacLeod of the Maricopa County
-Sheriff's office--and Sgt. Dan Pasquale's board in Fremont, California.
-Sysops posed as hackers, and swiftly garnered coteries of ardent users,
-who posted codes and loaded pirate software with abandon,
-and came to a sticky end.
-
-Sting boards, like other boards, are cheap to operate,
-very cheap by the standards of undercover police operations.
-Once accepted by the local underground, sysops will likely be
-invited into other pirate boards, where they can compile more dossiers.
-And when the sting is announced and the worst offenders arrested,
-the publicity is generally gratifying. The resultant paranoia
-in the underground--perhaps more justly described as a "deterrence effect"--
-tends to quell local lawbreaking for quite a while.
-
-Obviously police do not have to beat the underbrush for hackers.
-On the contrary, they can go trolling for them. Those caught
-can be grilled. Some become useful informants. They can lead
-the way to pirate boards all across the country.
-
-And boards all across the country showed the sticky
-fingerprints of Phrack, and of that loudest and most
-flagrant of all underground groups, the "Legion of Doom."
-
-The term "Legion of Doom" came from comic books. The Legion of Doom,
-a conspiracy of costumed super- villains headed by the chrome-domed
-criminal ultra- mastermind Lex Luthor, gave Superman a lot of four-color
-graphic trouble for a number of decades. Of course, Superman,
-that exemplar of Truth, Justice, and the American Way,
-always won in the long run. This didn't matter to the hacker Doomsters--
-"Legion of Doom" was not some thunderous and evil Satanic reference,
-it was not meant to be taken seriously. "Legion of Doom" came
-from funny-books and was supposed to be funny.
-
-"Legion of Doom" did have a good mouthfilling ring to it, though.
-It sounded really cool. Other groups, such as the "Farmers of Doom,"
-closely allied to LoD, recognized this grandiloquent quality,
-and made fun of it. There was even a hacker group called
-"Justice League of America," named after Superman's club
-of true-blue crimefighting superheros.
-
-But they didn't last; the Legion did.
-
-The original Legion of Doom, hanging out on Quasi Moto's Plovernet board,
-were phone phreaks. They weren't much into computers. "Lex Luthor" himself
-(who was under eighteen when he formed the Legion) was a COSMOS expert,
-COSMOS being the "Central System for Mainframe Operations,"
-a telco internal computer network. Lex would eventually become
-quite a dab hand at breaking into IBM mainframes, but although
-everyone liked Lex and admired his attitude, he was not considered
-a truly accomplished computer intruder. Nor was he the "mastermind"
-of the Legion of Doom--LoD were never big on formal leadership.
-As a regular on Plovernet and sysop of his "Legion of Doom BBS,"
-Lex was the Legion's cheerleader and recruiting officer.
-
-Legion of Doom began on the ruins of an earlier phreak group,
-The Knights of Shadow. Later, LoD was to subsume the personnel
-of the hacker group "Tribunal of Knowledge." People came and went
-constantly in LoD; groups split up or formed offshoots.
-
-Early on, the LoD phreaks befriended a few computer-intrusion
-enthusiasts, who became the associated "Legion of Hackers."
-Then the two groups conflated into the "Legion of Doom/Hackers,"
-or LoD/H. When the original "hacker" wing, Messrs. "Compu-Phreak"
-and "Phucked Agent 04," found other matters to occupy their time,
-the extra "/H" slowly atrophied out of the name; but by this time
-the phreak wing, Messrs. Lex Luthor, "Blue Archer," "Gary Seven,"
-"Kerrang Khan," "Master of Impact," "Silver Spy," "The Marauder,"
-and "The Videosmith," had picked up a plethora of intrusion
-expertise and had become a force to be reckoned with.
-
-LoD members seemed to have an instinctive understanding
-that the way to real power in the underground lay through
-covert publicity. LoD were flagrant. Not only was it one
-of the earliest groups, but the members took pains to widely
-distribute their illicit knowledge. Some LoD members,
-like "The Mentor," were close to evangelical about it.
-Legion of Doom Technical Journal began to show up on boards
-throughout the underground.
-
-LoD Technical Journal was named in cruel parody
-of the ancient and honored AT&T Technical Journal.
-The material in these two publications was quite similar--
-much of it, adopted from public journals and discussions
-in the telco community. And yet, the predatory attitude
-of LoD made even its most innocuous data seem deeply sinister;
-an outrage; a clear and present danger.
-
-To see why this should be, let's consider the following
-(invented) paragraphs, as a kind of thought experiment.
-
-(A) "W. Fred Brown, AT&T Vice President for
-Advanced Technical Development, testified May 8
-at a Washington hearing of the National Telecommunications
-and Information Administration (NTIA), regarding
-Bellcore's GARDEN project. GARDEN (Generalized
-Automatic Remote Distributed Electronic Network) is a
-telephone-switch programming tool that makes it possible
-to develop new telecom services, including hold-on-hold
-and customized message transfers, from any keypad terminal,
-within seconds. The GARDEN prototype combines centrex
-lines with a minicomputer using UNIX operating system software."
-
-(B) "Crimson Flash 512 of the Centrex Mobsters reports:
-D00dz, you wouldn't believe this GARDEN bullshit Bellcore's
-just come up with! Now you don't even need a lousy Commodore
-to reprogram a switch--just log on to GARDEN as a technician,
-and you can reprogram switches right off the keypad in any
-public phone booth! You can give yourself hold-on-hold
-and customized message transfers, and best of all,
-the thing is run off (notoriously insecure) centrex lines
-using--get this--standard UNIX software! Ha ha ha ha!"
-
-Message (A), couched in typical techno-bureaucratese,
-appears tedious and almost unreadable. (A) scarcely seems
-threatening or menacing. Message (B), on the other hand,
-is a dreadful thing, prima facie evidence of a dire conspiracy,
-definitely not the kind of thing you want your teenager reading.
-
-The INFORMATION, however, is identical. It is PUBLIC
-information, presented before the federal government in
-an open hearing. It is not "secret." It is not "proprietary."
-It is not even "confidential." On the contrary, the
-development of advanced software systems is a matter
-of great public pride to Bellcore.
-
-However, when Bellcore publicly announces a project of this kind,
-it expects a certain attitude from the public--something along
-the lines of GOSH WOW, YOU GUYS ARE GREAT, KEEP THAT UP, WHATEVER IT IS--
-certainly not cruel mimickry, one-upmanship and outrageous speculations
-about possible security holes.
-
-Now put yourself in the place of a policeman confronted by
-an outraged parent, or telco official, with a copy of Version (B).
-This well-meaning citizen, to his horror, has discovered
-a local bulletin-board carrying outrageous stuff like (B),
-which his son is examining with a deep and unhealthy interest.
-If (B) were printed in a book or magazine, you, as an American
-law enforcement officer, would know that it would take
-a hell of a lot of trouble to do anything about it;
-but it doesn't take technical genius to recognize that
-if there's a computer in your area harboring stuff like (B),
-there's going to be trouble.
-
-In fact, if you ask around, any computer-literate cop
-will tell you straight out that boards with stuff like (B)
-are the SOURCE of trouble. And the WORST source of trouble
-on boards are the ringleaders inventing and spreading stuff like (B).
-If it weren't for these jokers, there wouldn't BE any trouble.
-
-And Legion of Doom were on boards like nobody else.
-Plovernet. The Legion of Doom Board. The Farmers of Doom Board.
-Metal Shop. OSUNY. Blottoland. Private Sector. Atlantis.
-Digital Logic. Hell Phrozen Over.
-
-LoD members also ran their own boards. "Silver Spy" started
-his own board, "Catch-22," considered one of the heaviest around.
-So did "Mentor," with his "Phoenix Project." When they didn't run boards
-themselves, they showed up on other people's boards, to brag, boast,
-and strut. And where they themselves didn't go, their philes went,
-carrying evil knowledge and an even more evil attitude.
-
-As early as 1986, the police were under the vague impression
-that EVERYONE in the underground was Legion of Doom.
-LoD was never that large--considerably smaller than either
-"Metal Communications" or "The Administration," for instance--
-but LoD got tremendous press. Especially in Phrack,
-which at times read like an LoD fan magazine; and Phrack
-was everywhere, especially in the offices of telco security.
-You couldn't GET busted as a phone phreak, a hacker,
-or even a lousy codes kid or warez dood, without the cops
-asking if you were LoD.
-
-This was a difficult charge to deny, as LoD never
-distributed membership badges or laminated ID cards.
-If they had, they would likely have died out quickly,
-for turnover in their membership was considerable.
-LoD was less a high-tech street-gang than an ongoing
-state-of-mind. LoD was the Gang That Refused to Die.
-By 1990, LoD had RULED for ten years, and it seemed WEIRD
-to police that they were continually busting people who were
-only sixteen years old. All these teenage small-timers
-were pleading the tiresome hacker litany of "just curious,
-no criminal intent." Somewhere at the center of this
-conspiracy there had to be some serious adult masterminds,
-not this seemingly endless supply of myopic suburban
-white kids with high SATs and funny haircuts.
-
-There was no question that most any American hacker
-arrested would "know" LoD. They knew the handles
-of contributors to LoD Tech Journal, and were likely
-to have learned their craft through LoD boards and LoD activism.
-But they'd never met anyone from LoD. Even some of the
-rotating cadre who were actually and formally "in LoD"
-knew one another only by board-mail and pseudonyms.
-This was a highly unconventional profile for a criminal conspiracy.
-Computer networking, and the rapid evolution of the digital underground,
-made the situation very diffuse and confusing.
-
-Furthermore, a big reputation in the digital underground
-did not coincide with one's willingness to commit "crimes."
-Instead, reputation was based on cleverness and technical mastery.
-As a result, it often seemed that the HEAVIER the hackers were,
-the LESS likely they were to have committed any kind of common,
-easily prosecutable crime. There were some hackers who could really steal.
-And there were hackers who could really hack. But the two groups didn't seem
-to overlap much, if at all. For instance, most people in the underground
-looked up to "Emmanuel Goldstein" of 2600 as a hacker demigod.
-But Goldstein's publishing activities were entirely legal--
-Goldstein just printed dodgy stuff and talked about politics,
-he didn't even hack. When you came right down to it,
-Goldstein spent half his time complaining that computer security
-WASN'T STRONG ENOUGH and ought to be drastically improved
-across the board!
-
-Truly heavy-duty hackers, those with serious technical skills
-who had earned the respect of the underground, never stole money
-or abused credit cards. Sometimes they might abuse phone-codes--
-but often, they seemed to get all the free phone-time they wanted
-without leaving a trace of any kind.
-
-The best hackers, the most powerful and technically accomplished,
-were not professional fraudsters. They raided computers habitually,
-but wouldn't alter anything, or damage anything. They didn't even steal
-computer equipment--most had day-jobs messing with hardware,
-and could get all the cheap secondhand equipment they wanted.
-The hottest hackers, unlike the teenage wannabes, weren't snobs
-about fancy or expensive hardware. Their machines tended to be
-raw second-hand digital hot-rods full of custom add-ons that
-they'd cobbled together out of chickenwire, memory chips and spit.
-Some were adults, computer software writers and consultants by trade,
-and making quite good livings at it. Some of them ACTUALLY WORKED
-FOR THE PHONE COMPANY--and for those, the "hackers" actually found
-under the skirts of Ma Bell, there would be little mercy in 1990.
-
-It has long been an article of faith in the
-underground that the "best" hackers never get caught.
-They're far too smart, supposedly. They never get caught
-because they never boast, brag, or strut. These demigods
-may read underground boards (with a condescending smile),
-but they never say anything there. The "best" hackers,
-according to legend, are adult computer professionals,
-such as mainframe system administrators, who already know
-the ins and outs of their particular brand of security.
-Even the "best" hacker can't break in to just any computer at random:
-the knowledge of security holes is too specialized, varying widely
-with different software and hardware. But if people are employed to run,
-say, a UNIX mainframe or a VAX/VMS machine, then they tend to learn
-security from the inside out. Armed with this knowledge,
-they can look into most anybody else's UNIX or VMS
-without much trouble or risk, if they want to.
-And, according to hacker legend, of course they want to,
-so of course they do. They just don't make a big deal
-of what they've done. So nobody ever finds out.
-
-It is also an article of faith in the underground that
-professional telco people "phreak" like crazed weasels.
-OF COURSE they spy on Madonna's phone calls--I mean,
-WOULDN'T YOU? Of course they give themselves free long-
-distance--why the hell should THEY pay, they're running
-the whole shebang!
-
-It has, as a third matter, long been an article of faith
-that any hacker caught can escape serious punishment if
-he confesses HOW HE DID IT. Hackers seem to believe
-that governmental agencies and large corporations are
-blundering about in cyberspace like eyeless jellyfish
-or cave salamanders. They feel that these large
-but pathetically stupid organizations will proffer up
-genuine gratitude, and perhaps even a security post
-and a big salary, to the hot-shot intruder who will deign
-to reveal to them the supreme genius of his modus operandi.
-
-In the case of longtime LoD member "Control-C,"
-this actually happened, more or less. Control-C had led
-Michigan Bell a merry chase, and when captured in 1987,
-he turned out to be a bright and apparently physically
-harmless young fanatic, fascinated by phones. There was
-no chance in hell that Control-C would actually repay the
-enormous and largely theoretical sums in long-distance
-service that he had accumulated from Michigan Bell.
-He could always be indicted for fraud or computer-intrusion,
-but there seemed little real point in this--he hadn't
-physically damaged any computer. He'd just plead guilty,
-and he'd likely get the usual slap-on-the-wrist,
-and in the meantime it would be a big hassle for Michigan Bell
-just to bring up the case. But if kept on the payroll,
-he might at least keep his fellow hackers at bay.
-
-There were uses for him. For instance, a contrite
-Control-C was featured on Michigan Bell internal posters,
-sternly warning employees to shred their trash.
-He'd always gotten most of his best inside info from
-"trashing"--raiding telco dumpsters, for useful data
-indiscreetly thrown away. He signed these posters, too.
-Control-C had become something like a Michigan Bell mascot.
-And in fact, Control-C DID keep other hackers at bay.
-Little hackers were quite scared of Control-C and his
-heavy-duty Legion of Doom friends. And big hackers WERE
-his friends and didn't want to screw up his cushy situation.
-
-No matter what one might say of LoD, they did stick together.
-When "Wasp," an apparently genuinely malicious New York hacker,
-began crashing Bellcore machines, Control-C received swift volunteer
-help from "the Mentor" and the Georgia LoD wing made up of
-"The Prophet," "Urvile," and "Leftist." Using Mentor's Phoenix
-Project board to coordinate, the Doomsters helped telco security
-to trap Wasp, by luring him into a machine with a tap
-and line-trace installed. Wasp lost. LoD won! And my, did they brag.
-
-Urvile, Prophet and Leftist were well-qualified for this activity,
-probably more so even than the quite accomplished Control-C.
-The Georgia boys knew all about phone switching-stations.
-Though relative johnny-come-latelies in the Legion of Doom,
-they were considered some of LoD's heaviest guys,
-into the hairiest systems around. They had the good fortune
-to live in or near Atlanta, home of the sleepy and apparently
-tolerant BellSouth RBOC.
-
-As RBOC security went, BellSouth were "cake." US West (of Arizona,
-the Rockies and the Pacific Northwest) were tough and aggressive,
-probably the heaviest RBOC around. Pacific Bell, California's PacBell,
-were sleek, high-tech, and longtime veterans of the LA phone-phreak wars.
-NYNEX had the misfortune to run the New York City area, and were warily
-prepared for most anything. Even Michigan Bell, a division of the
-Ameritech RBOC, at least had the elementary sense to hire their own hacker
-as a useful scarecrow. But BellSouth, even though their corporate P.R.
-proclaimed them to have "Everything You Expect From a Leader," were pathetic.
-
-When rumor about LoD's mastery of Georgia's switching network got around
-to BellSouth through Bellcore and telco security scuttlebutt,
-they at first refused to believe it. If you paid serious attention
-to every rumor out and about these hacker kids, you would hear all kinds
-of wacko saucer-nut nonsense: that the National Security Agency
-monitored all American phone calls, that the CIA and DEA tracked
-traffic on bulletin-boards with word-analysis programs,
-that the Condor could start World War III from a payphone.
-
-If there were hackers into BellSouth switching-stations, then how come
-nothing had happened? Nothing had been hurt. BellSouth's machines
-weren't crashing. BellSouth wasn't suffering especially badly from fraud.
-BellSouth's customers weren't complaining. BellSouth was headquartered
-in Atlanta, ambitious metropolis of the new high-tech Sunbelt;
-and BellSouth was upgrading its network by leaps and bounds,
-digitizing the works left right and center. They could hardly be
-considered sluggish or naive. BellSouth's technical expertise
-was second to none, thank you kindly. But then came the Florida business.
-
-On June 13, 1989, callers to the Palm Beach County Probation Department,
-in Delray Beach, Florida, found themselves involved in a remarkable
-discussion with a phone-sex worker named "Tina" in New York State.
-Somehow, ANY call to this probation office near Miami was instantly
-and magically transported across state lines, at no extra charge to the user,
-to a pornographic phone-sex hotline hundreds of miles away!
-
-This practical joke may seem utterly hilarious at first hearing,
-and indeed there was a good deal of chuckling about it in
-phone phreak circles, including the Autumn 1989 issue of 2600.
-But for Southern Bell (the division of the BellSouth RBOC
-supplying local service for Florida, Georgia, North Carolina
-and South Carolina), this was a smoking gun. For the first time ever,
-a computer intruder had broken into a BellSouth central office
-switching station and re-programmed it!
-
-Or so BellSouth thought in June 1989. Actually, LoD members had been
-frolicking harmlessly in BellSouth switches since September 1987.
-The stunt of June 13--call-forwarding a number through manipulation
-of a switching station--was child's play for hackers as accomplished
-as the Georgia wing of LoD. Switching calls interstate sounded like
-a big deal, but it took only four lines of code to accomplish this.
-An easy, yet more discreet, stunt, would be to call-forward another
-number to your own house. If you were careful and considerate,
-and changed the software back later, then not a soul would know.
-Except you. And whoever you had bragged to about it.
-
-As for BellSouth, what they didn't know wouldn't hurt them.
-
-Except now somebody had blown the whole thing wide open, and BellSouth knew.
-
-A now alerted and considerably paranoid BellSouth began searching switches
-right and left for signs of impropriety, in that hot summer of 1989.
-No fewer than forty-two BellSouth employees were put on 12-hour shifts,
-twenty-four hours a day, for two solid months, poring over records
-and monitoring computers for any sign of phony access. These forty-two
-overworked experts were known as BellSouth's "Intrusion Task Force."
-
-What the investigators found astounded them. Proprietary telco databases
-had been manipulated: phone numbers had been created out of thin air,
-with no users' names and no addresses. And perhaps worst of all,
-no charges and no records of use. The new digital ReMOB (Remote Observation)
-diagnostic feature had been extensively tampered with--hackers had learned to
-reprogram ReMOB software, so that they could listen in on any switch-routed
-call at their leisure! They were using telco property to SPY!
-
-The electrifying news went out throughout law enforcement in 1989.
-It had never really occurred to anyone at BellSouth that their prized
-and brand-new digital switching-stations could be RE-PROGRAMMED.
-People seemed utterly amazed that anyone could have the nerve.
-Of course these switching stations were "computers," and everybody
-knew hackers liked to "break into computers:" but telephone people's
-computers were DIFFERENT from normal people's computers.
-
-The exact reason WHY these computers were "different" was
-rather ill-defined. It certainly wasn't the extent of their security.
-The security on these BellSouth computers was lousy; the AIMSX computers,
-for instance, didn't even have passwords. But there was no question that
-BellSouth strongly FELT that their computers were very different indeed.
-And if there were some criminals out there who had not gotten that message,
-BellSouth was determined to see that message taught.
-
-After all, a 5ESS switching station was no mere bookkeeping system for
-some local chain of florists. Public service depended on these stations.
-Public SAFETY depended on these stations.
-
-And hackers, lurking in there call-forwarding or ReMobbing, could spy
-on anybody in the local area! They could spy on telco officials!
-They could spy on police stations! They could spy on local offices
-of the Secret Service. . . .
-
-In 1989, electronic cops and hacker-trackers began using scrambler-phones
-and secured lines. It only made sense. There was no telling who was into
-those systems. Whoever they were, they sounded scary. This was some
-new level of antisocial daring. Could be West German hackers, in the pay
-of the KGB. That too had seemed a weird and farfetched notion,
-until Clifford Stoll had poked and prodded a sluggish Washington
-law-enforcement bureaucracy into investigating a computer intrusion
-that turned out to be exactly that--HACKERS, IN THE PAY OF THE KGB!
-Stoll, the systems manager for an Internet lab in Berkeley California,
-had ended up on the front page of the New Nork Times, proclaimed a national
-hero in the first true story of international computer espionage.
-Stoll's counterspy efforts, which he related in a bestselling book,
-The Cuckoo's Egg, in 1989, had established the credibility of `hacking'
-as a possible threat to national security. The United States Secret Service
-doesn't mess around when it suspects a possible action by a foreign
-intelligence apparat.
-
-The Secret Service scrambler-phones and secured lines put
-a tremendous kink in law enforcement's ability to operate freely;
-to get the word out, cooperate, prevent misunderstandings.
-Nevertheless, 1989 scarcely seemed the time for half-measures.
-If the police and Secret Service themselves were not operationally secure,
-then how could they reasonably demand measures of security from
-private enterprise? At least, the inconvenience made people aware
-of the seriousness of the threat.
-
-If there was a final spur needed to get the police off the dime,
-it came in the realization that the emergency 911 system was vulnerable.
-The 911 system has its own specialized software, but it is run on the same
-digital switching systems as the rest of the telephone network.
-911 is not physically different from normal telephony. But it is
-certainly culturally different, because this is the area of
-telephonic cyberspace reserved for the police and emergency services.
-
-Your average policeman may not know much about hackers or phone-phreaks.
-Computer people are weird; even computer COPS are rather weird;
-the stuff they do is hard to figure out. But a threat to the 911 system
-is anything but an abstract threat. If the 911 system goes, people can die.
-
-Imagine being in a car-wreck, staggering to a phone-booth,
-punching 911 and hearing "Tina" pick up the phone-sex line
-somewhere in New York! The situation's no longer comical, somehow.
-
-And was it possible? No question. Hackers had attacked 911
-systems before. Phreaks can max-out 911 systems just by siccing
-a bunch of computer-modems on them in tandem, dialling them over
-and over until they clog. That's very crude and low-tech,
-but it's still a serious business.
-
-The time had come for action. It was time to take stern measures
-with the underground. It was time to start picking up the dropped threads,
-the loose edges, the bits of braggadocio here and there; it was time to get
-on the stick and start putting serious casework together. Hackers weren't
-"invisible." They THOUGHT they were invisible; but the truth was,
-they had just been tolerated too long.
-
-Under sustained police attention in the summer of '89, the digital
-underground began to unravel as never before.
-
-The first big break in the case came very early on: July 1989,
-the following month. The perpetrator of the "Tina" switch was caught,
-and confessed. His name was "Fry Guy," a 16-year-old in Indiana.
-Fry Guy had been a very wicked young man.
-
-Fry Guy had earned his handle from a stunt involving French fries.
-Fry Guy had filched the log-in of a local MacDonald's manager
-and had logged-on to the MacDonald's mainframe on the Sprint
-Telenet system. Posing as the manager, Fry Guy had altered
-MacDonald's records, and given some teenage hamburger-flipping
-friends of his, generous raises. He had not been caught.
-
-Emboldened by success, Fry Guy moved on to credit-card abuse.
-Fry Guy was quite an accomplished talker; with a gift for
-"social engineering." If you can do "social engineering"
---fast-talk, fake-outs, impersonation, conning, scamming--
-then card abuse comes easy. (Getting away with it in
-the long run is another question).
-
-Fry Guy had run across "Urvile" of the Legion of Doom
-on the ALTOS Chat board in Bonn, Germany. ALTOS Chat
-was a sophisticated board, accessible through globe-spanning
-computer networks like BITnet, Tymnet, and Telenet.
-ALTOS was much frequented by members of Germany's
-Chaos Computer Club. Two Chaos hackers who hung out on ALTOS,
-"Jaeger" and "Pengo," had been the central villains of
-Clifford Stoll's Cuckoo's Egg case: consorting in East Berlin
-with a spymaster from the KGB, and breaking into American
-computers for hire, through the Internet.
-
-When LoD members learned the story of Jaeger's depredations
-from Stoll's book, they were rather less than impressed,
-technically speaking. On LoD's own favorite board of the moment,
-"Black Ice," LoD members bragged that they themselves could have done
-all the Chaos break-ins in a week flat! Nevertheless, LoD were grudgingly
-impressed by the Chaos rep, the sheer hairy-eyed daring of hash-smoking
-anarchist hackers who had rubbed shoulders with the fearsome big-boys
-of international Communist espionage. LoD members sometimes traded
-bits of knowledge with friendly German hackers on ALTOS--phone numbers
-for vulnerable VAX/VMS computers in Georgia, for instance.
-Dutch and British phone phreaks, and the Australian clique of
-"Phoenix," "Nom," and "Electron," were ALTOS regulars, too.
-In underground circles, to hang out on ALTOS was considered
-the sign of an elite dude, a sophisticated hacker of the
-international digital jet-set.
-
-Fry Guy quickly learned how to raid information from credit-card
-consumer-reporting agencies. He had over a hundred stolen credit-card
-numbers in his notebooks, and upwards of a thousand swiped long-distance
-access codes. He knew how to get onto Altos, and how to talk the talk of
-the underground convincingly. He now wheedled knowledge of switching-station
-tricks from Urvile on the ALTOS system.
-
-Combining these two forms of knowledge enabled Fry Guy to bootstrap
-his way up to a new form of wire-fraud. First, he'd snitched credit card
-numbers from credit-company computers. The data he copied included names,
-addresses and phone numbers of the random card-holders.
-
-Then Fry Guy, impersonating a card-holder, called up Western Union
-and asked for a cash advance on "his" credit card. Western Union,
-as a security guarantee, would call the customer back, at home,
-to verify the transaction.
-
-But, just as he had switched the Florida probation office to "Tina"
-in New York, Fry Guy switched the card-holder's number to a local pay-phone.
-There he would lurk in wait, muddying his trail by routing and re-routing
-the call, through switches as far away as Canada. When the call came through,
-he would boldly "social-engineer," or con, the Western Union people, pretending
-to be the legitimate card-holder. Since he'd answered the proper phone number,
-the deception was not very hard. Western Union's money was then shipped to
-a confederate of Fry Guy's in his home town in Indiana.
-
-Fry Guy and his cohort, using LoD techniques, stole six thousand dollars
-from Western Union between December 1988 and July 1989. They also dabbled
-in ordering delivery of stolen goods through card-fraud. Fry Guy
-was intoxicated with success. The sixteen-year-old fantasized wildly
-to hacker rivals, boasting that he'd used rip-off money to hire himself
-a big limousine, and had driven out-of-state with a groupie from
-his favorite heavy-metal band, Motley Crue.
-
-Armed with knowledge, power, and a gratifying stream of free money,
-Fry Guy now took it upon himself to call local representatives
-of Indiana Bell security, to brag, boast, strut, and utter
-tormenting warnings that his powerful friends in the notorious
-Legion of Doom could crash the national telephone network.
-Fry Guy even named a date for the scheme: the Fourth of July,
-a national holiday.
-
-This egregious example of the begging-for-arrest syndrome was shortly
-followed by Fry Guy's arrest. After the Indiana telephone company figured
-out who he was, the Secret Service had DNRs--Dialed Number Recorders--
-installed on his home phone lines. These devices are not taps, and can't
-record the substance of phone calls, but they do record the phone numbers
-of all calls going in and out. Tracing these numbers showed Fry Guy's
-long-distance code fraud, his extensive ties to pirate bulletin boards,
-and numerous personal calls to his LoD friends in Atlanta. By July 11,
-1989, Prophet, Urvile and Leftist also had Secret Service DNR
-"pen registers" installed on their own lines.
-
-The Secret Service showed up in force at Fry Guy's house on July 22, 1989,
-to the horror of his unsuspecting parents. The raiders were led by
-a special agent from the Secret Service's Indianapolis office.
-However, the raiders were accompanied and advised by Timothy M. Foley
-of the Secret Service's Chicago office (a gentleman about whom
-we will soon be hearing a great deal).
-
-Following federal computer-crime techniques that had been standard
-since the early 1980s, the Secret Service searched the house thoroughly,
-and seized all of Fry Guy's electronic equipment and notebooks.
-All Fry Guy's equipment went out the door in the custody of the
-Secret Service, which put a swift end to his depredations.
-
-The USSS interrogated Fry Guy at length. His case was put in the charge
-of Deborah Daniels, the federal US Attorney for the Southern District
-of Indiana. Fry Guy was charged with eleven counts of computer fraud,
-unauthorized computer access, and wire fraud. The evidence was thorough
-and irrefutable. For his part, Fry Guy blamed his corruption on the
-Legion of Doom and offered to testify against them.
-
-Fry Guy insisted that the Legion intended to crash the phone system
-on a national holiday. And when AT&T crashed on Martin Luther King Day,
-1990, this lent a credence to his claim that genuinely alarmed telco
-security and the Secret Service.
-
-Fry Guy eventually pled guilty on May 31, 1990. On September 14,
-he was sentenced to forty-four months' probation and four hundred hours'
-community service. He could have had it much worse; but it made sense
-to prosecutors to take it easy on this teenage minor, while zeroing
-in on the notorious kingpins of the Legion of Doom.
-
-But the case against LoD had nagging flaws. Despite the best effort
-of investigators, it was impossible to prove that the Legion had crashed
-the phone system on January 15, because they, in fact, hadn't done so.
-The investigations of 1989 did show that certain members of
-the Legion of Doom had achieved unprecedented power over the telco
-switching stations, and that they were in active conspiracy
-to obtain more power yet. Investigators were privately convinced
-that the Legion of Doom intended to do awful things with this knowledge,
-but mere evil intent was not enough to put them in jail.
-
-And although the Atlanta Three--Prophet, Leftist, and especially Urvile--
-had taught Fry Guy plenty, they were not themselves credit-card fraudsters.
-The only thing they'd "stolen" was long-distance service--and since they'd
-done much of that through phone-switch manipulation, there was no easy way
-to judge how much they'd "stolen," or whether this practice was even "theft"
-of any easily recognizable kind.
-
-Fry Guy's theft of long-distance codes had cost the phone companies plenty.
-The theft of long-distance service may be a fairly theoretical "loss,"
-but it costs genuine money and genuine time to delete all those stolen codes,
-and to re-issue new codes to the innocent owners of those corrupted codes.
-The owners of the codes themselves are victimized, and lose time and money
-and peace of mind in the hassle. And then there were the credit-card victims
-to deal with, too, and Western Union. When it came to rip-off, Fry Guy was
-far more of a thief than LoD. It was only when it came to actual computer
-expertise that Fry Guy was small potatoes.
-
-The Atlanta Legion thought most "rules" of cyberspace were for rodents
-and losers, but they DID have rules. THEY NEVER CRASHED ANYTHING,
-AND THEY NEVER TOOK MONEY. These were rough rules-of-thumb, and
-rather dubious principles when it comes to the ethical subtleties
-of cyberspace, but they enabled the Atlanta Three to operate with
-a relatively clear conscience (though never with peace of mind).
-
-If you didn't hack for money, if you weren't robbing people of actual funds
---money in the bank, that is-- then nobody REALLY got hurt, in LoD's opinion.
-"Theft of service" was a bogus issue, and "intellectual property" was
-a bad joke. But LoD had only elitist contempt for rip-off artists,
-"leechers," thieves. They considered themselves clean. In their opinion,
-if you didn't smash-up or crash any systems --(well, not on purpose, anyhow--
-accidents can happen, just ask Robert Morris) then it was very unfair
-to call you a "vandal" or a "cracker." When you were hanging out on-line
-with your "pals" in telco security, you could face them down from the higher
-plane of hacker morality. And you could mock the police from the supercilious
-heights of your hacker's quest for pure knowledge.
-
-But from the point of view of law enforcement and telco security, however,
-Fry Guy was not really dangerous. The Atlanta Three WERE dangerous.
-It wasn't the crimes they were committing, but the DANGER,
-the potential hazard, the sheer TECHNICAL POWER LoD had accumulated,
-that had made the situation untenable. Fry Guy was not LoD.
-He'd never laid eyes on anyone in LoD; his only contacts with them
-had been electronic. Core members of the Legion of Doom tended to meet
-physically for conventions every year or so, to get drunk, give each other
-the hacker high-sign, send out for pizza and ravage hotel suites.
-Fry Guy had never done any of this. Deborah Daniels assessed Fry Guy
-accurately as "an LoD wannabe."
-
-Nevertheless Fry Guy's crimes would be directly attributed to LoD
-in much future police propaganda. LoD would be described as
-"a closely knit group" involved in "numerous illegal activities"
-including "stealing and modifying individual credit histories,"
-and "fraudulently obtaining money and property." Fry Guy did this,
-but the Atlanta Three didn't; they simply weren't into theft,
-but rather intrusion. This caused a strange kink in
-the prosecution's strategy. LoD were accused of
-"disseminating information about attacking computers
-to other computer hackers in an effort to shift the focus
-of law enforcement to those other hackers and away from the Legion of Doom."
-
-This last accusation (taken directly from a press release by the Chicago
-Computer Fraud and Abuse Task Force) sounds particularly far-fetched.
-One might conclude at this point that investigators would have been
-well-advised to go ahead and "shift their focus" from the "Legion of Doom."
-Maybe they SHOULD concentrate on "those other hackers"--the ones who were
-actually stealing money and physical objects.
-
-But the Hacker Crackdown of 1990 was not a simple policing action.
-It wasn't meant just to walk the beat in cyberspace--it was a CRACKDOWN,
-a deliberate attempt to nail the core of the operation, to send a dire
-and potent message that would settle the hash of the digital underground
-for good.
-
-By this reasoning, Fry Guy wasn't much more than the electronic equivalent
-of a cheap streetcorner dope dealer. As long as the masterminds of LoD were
-still flagrantly operating, pushing their mountains of illicit knowledge
-right and left, and whipping up enthusiasm for blatant lawbreaking,
-then there would be an INFINITE SUPPLY of Fry Guys.
-
-Because LoD were flagrant, they had left trails everywhere,
-to be picked up by law enforcement in New York, Indiana,
-Florida, Texas, Arizona, Missouri, even Australia.
-But 1990's war on the Legion of Doom was led out of Illinois,
-by the Chicago Computer Fraud and Abuse Task Force.
-
-#
-
-The Computer Fraud and Abuse Task Force, led by federal prosecutor
-William J. Cook, had started in 1987 and had swiftly become one
-of the most aggressive local "dedicated computer-crime units."
-Chicago was a natural home for such a group. The world's first
-computer bulletin-board system had been invented in Illinois.
-The state of Illinois had some of the nation's first and sternest
-computer crime laws. Illinois State Police were markedly alert
-to the possibilities of white-collar crime and electronic fraud.
-
-And William J. Cook in particular was a rising star in
-electronic crime-busting. He and his fellow federal prosecutors
-at the U.S. Attorney's office in Chicago had a tight relation
-with the Secret Service, especially go-getting Chicago-based agent
-Timothy Foley. While Cook and his Department of Justice colleagues
-plotted strategy, Foley was their man on the street.
-
-Throughout the 1980s, the federal government had given prosecutors
-an armory of new, untried legal tools against computer crime.
-Cook and his colleagues were pioneers in the use of these new statutes
-in the real-life cut-and-thrust of the federal courtroom.
-
-On October 2, 1986, the US Senate had passed the
-"Computer Fraud and Abuse Act" unanimously, but there
-were pitifully few convictions under this statute.
-Cook's group took their name from this statute,
-since they were determined to transform this powerful but
-rather theoretical Act of Congress into a real-life engine
-of legal destruction against computer fraudsters and scofflaws.
-
-It was not a question of merely discovering crimes,
-investigating them, and then trying and punishing their
-perpetrators. The Chicago unit, like most everyone else
-in the business, already KNEW who the bad guys were:
-the Legion of Doom and the writers and editors of Phrack.
-The task at hand was to find some legal means of putting
-these characters away.
-
-This approach might seem a bit dubious, to someone not
-acquainted with the gritty realities of prosecutorial work.
-But prosecutors don't put people in jail for crimes
-they have committed; they put people in jail for crimes
-they have committed THAT CAN BE PROVED IN COURT.
-Chicago federal police put Al Capone in prison
-for income-tax fraud. Chicago is a big town,
-with a rough-and-ready bare-knuckle tradition
-on both sides of the law.
-
-Fry Guy had broken the case wide open and alerted telco security
-to the scope of the problem. But Fry Guy's crimes would not
-put the Atlanta Three behind bars--much less the wacko underground
-journalists of Phrack. So on July 22, 1989, the same day that
-Fry Guy was raided in Indiana, the Secret Service descended upon
-the Atlanta Three.
-
-This was likely inevitable. By the summer of 1989, law enforcement
-were closing in on the Atlanta Three from at least six directions at once.
-First, there were the leads from Fry Guy, which had led to the DNR registers
-being installed on the lines of the Atlanta Three. The DNR evidence alone
-would have finished them off, sooner or later.
-
-But second, the Atlanta lads were already well-known to Control-C
-and his telco security sponsors. LoD's contacts with telco security
-had made them overconfident and even more boastful than usual;
-they felt that they had powerful friends in high places,
-and that they were being openly tolerated by telco security.
-But BellSouth's Intrusion Task Force were hot on the trail of LoD
-and sparing no effort or expense.
-
-The Atlanta Three had also been identified by name and listed
-on the extensive anti-hacker files maintained, and retailed for pay,
-by private security operative John Maxfield of Detroit.
-Maxfield, who had extensive ties to telco security
-and many informants in the underground, was a bete noire
-of the Phrack crowd, and the dislike was mutual.
-
-
-The Atlanta Three themselves had written articles for Phrack.
-This boastful act could not possibly escape telco and law enforcement
-attention.
-
-"Knightmare," a high-school age hacker from Arizona,
-was a close friend and disciple of Atlanta LoD,
-but he had been nabbed by the formidable Arizona
-Organized Crime and Racketeering Unit. Knightmare was
-on some of LoD's favorite boards--"Black Ice" in particular--
-and was privy to their secrets. And to have Gail Thackeray,
-the Assistant Attorney General of Arizona, on one's trail
-was a dreadful peril for any hacker.
-
-And perhaps worst of all, Prophet had committed a major blunder
-by passing an illicitly copied BellSouth computer-file to Knight Lightning,
-who had published it in Phrack. This, as we will see, was an act of dire
-consequence for almost everyone concerned.
-
-On July 22, 1989, the Secret Service showed up at the Leftist's house,
-where he lived with his parents. A massive squad of some twenty officers
-surrounded the building: Secret Service, federal marshals, local police,
-possibly BellSouth telco security; it was hard to tell in the crush.
-Leftist's dad, at work in his basement office, first noticed
-a muscular stranger in plain clothes crashing through the
-back yard with a drawn pistol. As more strangers poured
-into the house, Leftist's dad naturally assumed there was
-an armed robbery in progress.
-
-Like most hacker parents, Leftist's mom and dad had only the vaguest
-notions of what their son had been up to all this time. Leftist had
-a day-job repairing computer hardware. His obsession with computers
-seemed a bit odd, but harmless enough, and likely to produce a well-
-paying career. The sudden, overwhelming raid left Leftist's
-parents traumatized.
-
-The Leftist himself had been out after work with his co-workers,
-surrounding a couple of pitchers of margaritas. As he came trucking
-on tequila-numbed feet up the pavement, toting a bag full of floppy-disks,
-he noticed a large number of unmarked cars parked in his driveway.
-All the cars sported tiny microwave antennas.
-
-The Secret Service had knocked the front door off its hinges,
-almost flattening his mom.
-
-Inside, Leftist was greeted by Special Agent James Cool
-of the US Secret Service, Atlanta office. Leftist was flabbergasted.
-He'd never met a Secret Service agent before. He could not imagine
-that he'd ever done anything worthy of federal attention.
-He'd always figured that if his activities became intolerable,
-one of his contacts in telco security would give him a private
-phone-call and tell him to knock it off.
-
-But now Leftist was pat-searched for weapons by grim professionals,
-and his bag of floppies was quickly seized. He and his parents were
-all shepherded into separate rooms and grilled at length as a score
-of officers scoured their home for anything electronic.
-
-Leftist was horrified as his treasured IBM AT personal computer
-with its forty-meg hard disk, and his recently purchased 80386 IBM-clone
-with a whopping hundred-meg hard disk, both went swiftly out the door
-in Secret Service custody. They also seized all his disks, all his notebooks,
-and a tremendous booty in dogeared telco documents that Leftist had snitched
-out of trash dumpsters.
-
-Leftist figured the whole thing for a big misunderstanding.
-He'd never been into MILITARY computers. He wasn't a SPY or a COMMUNIST.
-He was just a good ol' Georgia hacker, and now he just wanted all these
-people out of the house. But it seemed they wouldn't go until he made
-some kind of statement.
-
-And so, he levelled with them.
-
-And that, Leftist said later from his federal prison camp in Talladega,
-Alabama, was a big mistake. The Atlanta area was unique,
-in that it had three members of the Legion of Doom who actually
-occupied more or less the same physical locality. Unlike the rest
-of LoD, who tended to associate by phone and computer,
-Atlanta LoD actually WERE "tightly knit." It was no real
-surprise that the Secret Service agents apprehending Urvile
-at the computer-labs at Georgia Tech, would discover Prophet
-with him as well.
-
-Urvile, a 21-year-old Georgia Tech student in polymer chemistry,
-posed quite a puzzling case for law enforcement. Urvile--also known
-as "Necron 99," as well as other handles, for he tended to change his
-cover-alias about once a month--was both an accomplished hacker
-and a fanatic simulation-gamer.
-
-Simulation games are an unusual hobby; but then hackers are unusual people,
-and their favorite pastimes tend to be somewhat out of the ordinary.
-The best-known American simulation game is probably "Dungeons & Dragons,"
-a multi-player parlor entertainment played with paper, maps, pencils,
-statistical tables and a variety of oddly-shaped dice. Players pretend
-to be heroic characters exploring a wholly-invented fantasy world.
-The fantasy worlds of simulation gaming are commonly pseudo-medieval,
-involving swords and sorcery--spell-casting wizards, knights in armor,
-unicorns and dragons, demons and goblins.
-
-Urvile and his fellow gamers preferred their fantasies highly technological.
-They made use of a game known as "G.U.R.P.S.," the "Generic Universal Role
-Playing System," published by a company called Steve Jackson Games (SJG).
-
-"G.U.R.P.S." served as a framework for creating a wide variety of artificial
-fantasy worlds. Steve Jackson Games published a smorgasboard of books,
-full of detailed information and gaming hints, which were used to flesh-out
-many different fantastic backgrounds for the basic GURPS framework.
-Urvile made extensive use of two SJG books called GURPS High-Tech
-and GURPS Special Ops.
-
-In the artificial fantasy-world of GURPS Special Ops,
-players entered a modern fantasy of intrigue and international espionage.
-On beginning the game, players started small and powerless,
-perhaps as minor-league CIA agents or penny-ante arms dealers.
-But as players persisted through a series of game sessions
-(game sessions generally lasted for hours, over long,
-elaborate campaigns that might be pursued for months on end)
-then they would achieve new skills, new knowledge, new power.
-They would acquire and hone new abilities, such as marksmanship,
-karate, wiretapping, or Watergate burglary. They could also win
-various kinds of imaginary booty, like Berettas, or martini shakers,
-or fast cars with ejection seats and machine-guns under the headlights.
-
-As might be imagined from the complexity of these games,
-Urvile's gaming notes were very detailed and extensive.
-Urvile was a "dungeon-master," inventing scenarios
-for his fellow gamers, giant simulated adventure-puzzles
-for his friends to unravel. Urvile's game notes covered
-dozens of pages with all sorts of exotic lunacy, all about
-ninja raids on Libya and break-ins on encrypted Red Chinese supercomputers.
-His notes were written on scrap-paper and kept in loose-leaf binders.
-
-The handiest scrap paper around Urvile's college digs were the many pounds of
-BellSouth printouts and documents that he had snitched out of telco dumpsters.
-His notes were written on the back of misappropriated telco property.
-Worse yet, the gaming notes were chaotically interspersed with Urvile's
-hand-scrawled records involving ACTUAL COMPUTER INTRUSIONS that he
-had committed.
-
-Not only was it next to impossible to tell Urvile's fantasy game-notes
-from cyberspace "reality," but Urvile himself barely made this distinction.
-It's no exaggeration to say that to Urvile it was ALL a game. Urvile was
-very bright, highly imaginative, and quite careless of other people's notions
-of propriety. His connection to "reality" was not something to which he paid
-a great deal of attention.
-
-Hacking was a game for Urvile. It was an amusement he was carrying out,
-it was something he was doing for fun. And Urvile was an obsessive young man.
-He could no more stop hacking than he could stop in the middle of
-a jigsaw puzzle, or stop in the middle of reading a Stephen Donaldson
-fantasy trilogy. (The name "Urvile" came from a best-selling Donaldson novel.)
-
-Urvile's airy, bulletproof attitude seriously annoyed his interrogators.
-First of all, he didn't consider that he'd done anything wrong.
-There was scarcely a shred of honest remorse in him. On the contrary,
-he seemed privately convinced that his police interrogators were operating
-in a demented fantasy-world all their own. Urvile was too polite
-and well-behaved to say this straight-out, but his reactions were askew
-and disquieting.
-
-For instance, there was the business about LoD's ability
-to monitor phone-calls to the police and Secret Service.
-Urvile agreed that this was quite possible, and posed
-no big problem for LoD. In fact, he and his friends
-had kicked the idea around on the "Black Ice" board,
-much as they had discussed many other nifty notions,
-such as building personal flame-throwers and jury-rigging
-fistfulls of blasting-caps. They had hundreds of dial-up numbers
-for government agencies that they'd gotten through scanning Atlanta phones,
-or had pulled from raided VAX/VMS mainframe computers.
-
-Basically, they'd never gotten around to listening in on the cops
-because the idea wasn't interesting enough to bother with.
-Besides, if they'd been monitoring Secret Service phone calls,
-obviously they'd never have been caught in the first place. Right?
-
-The Secret Service was less than satisfied with this rapier-like hacker logic.
-
-Then there was the issue of crashing the phone system. No problem,
-Urvile admitted sunnily. Atlanta LoD could have shut down phone service
-all over Atlanta any time they liked. EVEN THE 911 SERVICE?
-Nothing special about that, Urvile explained patiently.
-Bring the switch to its knees, with say the UNIX "makedir" bug,
-and 911 goes down too as a matter of course. The 911 system
-wasn't very interesting, frankly. It might be tremendously
-interesting to cops (for odd reasons of their own), but as
-technical challenges went, the 911 service was yawnsville.
-
-So of course the Atlanta Three could crash service.
-They probably could have crashed service all over
-BellSouth territory, if they'd worked at it for a while.
-But Atlanta LoD weren't crashers. Only losers and rodents
-were crashers. LoD were ELITE.
-
-Urvile was privately convinced that sheer technical
-expertise could win him free of any kind of problem.
-As far as he was concerned, elite status in the digital
-underground had placed him permanently beyond the intellectual
-grasp of cops and straights. Urvile had a lot to learn.
-
-Of the three LoD stalwarts, Prophet was in the most direct trouble.
-Prophet was a UNIX programming expert who burrowed in and out
-of the Internet as a matter of course. He'd started his hacking
-career at around age 14, meddling with a UNIX mainframe system
-at the University of North Carolina.
-
-Prophet himself had written the handy Legion of Doom
-file "UNIX Use and Security From the Ground Up."
-UNIX (pronounced "you-nicks") is a powerful,
-flexible computer operating-system, for multi-user,
-multi-tasking computers. In 1969, when UNIX was created
-in Bell Labs, such computers were exclusive to large
-corporations and universities, but today UNIX is run
-on thousands of powerful home machines. UNIX was
-particularly well-suited to telecommunications programming,
-and had become a standard in the field. Naturally, UNIX
-also became a standard for the elite hacker and phone phreak.
-Lately, Prophet had not been so active as Leftist and Urvile,
-but Prophet was a recidivist. In 1986, when he was eighteen,
-Prophet had been convicted of "unauthorized access
-to a computer network" in North Carolina. He'd been
-discovered breaking into the Southern Bell Data Network,
-a UNIX-based internal telco network supposedly closed to the public.
-He'd gotten a typical hacker sentence: six months suspended,
-120 hours community service, and three years' probation.
-
-After that humiliating bust, Prophet had gotten rid of most of his
-tonnage of illicit phreak and hacker data, and had tried to go straight.
-He was, after all, still on probation. But by the autumn of 1988,
-the temptations of cyberspace had proved too much for young Prophet,
-and he was shoulder-to-shoulder with Urvile and Leftist into some
-of the hairiest systems around.
-
-In early September 1988, he'd broken into BellSouth's centralized
-automation system, AIMSX or "Advanced Information Management System."
-AIMSX was an internal business network for BellSouth, where telco
-employees stored electronic mail, databases, memos, and calendars,
-and did text processing. Since AIMSX did not have public dial-ups,
-it was considered utterly invisible to the public, and was not well-secured
---it didn't even require passwords. Prophet abused an account known
-as "waa1," the personal account of an unsuspecting telco employee.
-Disguised as the owner of waa1, Prophet made about ten visits to AIMSX.
-
-Prophet did not damage or delete anything in the system.
-His presence in AIMSX was harmless and almost invisible.
-But he could not rest content with that.
-
-One particular piece of processed text on AIMSX was a telco document
-known as "Bell South Standard Practice 660-225-104SV Control Office
-Administration of Enhanced 911 Services for Special Services
-and Major Account Centers dated March 1988."
-
-Prophet had not been looking for this document. It was merely one
-among hundreds of similar documents with impenetrable titles.
-However, having blundered over it in the course of his illicit
-wanderings through AIMSX, he decided to take it with him as a trophy.
-It might prove very useful in some future boasting, bragging,
-and strutting session. So, some time in September 1988,
-Prophet ordered the AIMSX mainframe computer to copy this document
-(henceforth called simply called "the E911 Document") and to transfer
-this copy to his home computer.
-
-No one noticed that Prophet had done this. He had "stolen"
-the E911 Document in some sense, but notions of property
-in cyberspace can be tricky. BellSouth noticed nothing wrong,
-because BellSouth still had their original copy. They had not
-been "robbed" of the document itself. Many people were supposed
-to copy this document--specifically, people who worked for the
-nineteen BellSouth "special services and major account centers,"
-scattered throughout the Southeastern United States. That was
-what it was for, why it was present on a computer network
-in the first place: so that it could be copied and read--
-by telco employees. But now the data had been copied
-by someone who wasn't supposed to look at it.
-
-Prophet now had his trophy. But he further decided to store
-yet another copy of the E911 Document on another person's computer.
-This unwitting person was a computer enthusiast named Richard Andrews
-who lived near Joliet, Illinois. Richard Andrews was a UNIX programmer
-by trade, and ran a powerful UNIX board called "Jolnet," in the basement
-of his house.
-
-Prophet, using the handle "Robert Johnson," had obtained an account
-on Richard Andrews' computer. And there he stashed the E911 Document,
-by storing it in his own private section of Andrews' computer.
-
-Why did Prophet do this? If Prophet had eliminated the E911 Document
-from his own computer, and kept it hundreds of miles away, on another machine, under an
-alias, then he might have been fairly safe from discovery and prosecution--
-although his sneaky action had certainly put the unsuspecting Richard Andrews
-at risk.
-
-But, like most hackers, Prophet was a pack-rat for illicit data.
-When it came to the crunch, he could not bear to part from his trophy.
-When Prophet's place in Decatur, Georgia was raided in July 1989,
-there was the E911 Document, a smoking gun. And there was Prophet
-in the hands of the Secret Service, doing his best to "explain."
-
-Our story now takes us away from the Atlanta Three and their raids
-of the Summer of 1989. We must leave Atlanta Three "cooperating fully"
-with their numerous investigators. And all three of them did cooperate,
-as their Sentencing Memorandum from the US District Court of the
-Northern Division of Georgia explained--just before all three of them
-were sentenced to various federal prisons in November 1990.
-
-We must now catch up on the other aspects of the war on the Legion of Doom.
-The war on the Legion was a war on a network--in fact, a network of three
-networks, which intertwined and interrelated in a complex fashion.
-The Legion itself, with Atlanta LoD, and their hanger-on Fry Guy,
-were the first network. The second network was Phrack magazine,
-with its editors and contributors.
-
-The third network involved the electronic circle around a hacker
-known as "Terminus."
-
-The war against these hacker networks was carried out by
-a law enforcement network. Atlanta LoD and Fry Guy
-were pursued by USSS agents and federal prosecutors in Atlanta,
-Indiana, and Chicago. "Terminus" found himself pursued by USSS
-and federal prosecutors from Baltimore and Chicago. And the war
-against Phrack was almost entirely a Chicago operation.
-
-The investigation of Terminus involved a great deal of energy,
-mostly from the Chicago Task Force, but it was to be the least-known
-and least-publicized of the Crackdown operations. Terminus, who lived
-in Maryland, was a UNIX programmer and consultant, fairly well-known
-(under his given name) in the UNIX community, as an acknowledged expert
-on AT&T minicomputers. Terminus idolized AT&T, especially Bellcore,
-and longed for public recognition as a UNIX expert; his highest ambition
-was to work for Bell Labs.
-
-But Terminus had odd friends and a spotted history.
-Terminus had once been the subject of an admiring interview
-in Phrack (Volume II, Issue 14, Phile 2--dated May 1987).
-In this article, Phrack co-editor Taran King described
-"Terminus" as an electronics engineer, 5'9", brown-haired,
-born in 1959--at 28 years old, quite mature for a hacker.
-
-Terminus had once been sysop of a phreak/hack underground board
-called "MetroNet," which ran on an Apple II. Later he'd replaced
-"MetroNet" with an underground board called "MegaNet,"
-specializing in IBMs. In his younger days, Terminus had written
-one of the very first and most elegant code-scanning programs
-for the IBM-PC. This program had been widely distributed
-in the underground. Uncounted legions of PC-owning phreaks and
-hackers had used Terminus's scanner program to rip-off telco codes.
-This feat had not escaped the attention of telco security;
-it hardly could, since Terminus's earlier handle, "Terminal Technician,"
-was proudly written right on the program.
-
-When he became a full-time computer professional
-(specializing in telecommunications programming),
-he adopted the handle Terminus, meant to indicate that he
-had "reached the final point of being a proficient hacker."
-He'd moved up to the UNIX-based "Netsys" board on an AT&T computer,
-with four phone lines and an impressive 240 megs of storage.
-"Netsys" carried complete issues of Phrack, and Terminus was
-quite friendly with its publishers, Taran King and Knight Lightning.
-
-In the early 1980s, Terminus had been a regular on Plovernet,
-Pirate-80, Sherwood Forest and Shadowland, all well-known pirate boards,
-all heavily frequented by the Legion of Doom. As it happened, Terminus
-was never officially "in LoD," because he'd never been given the official
-LoD high-sign and back-slap by Legion maven Lex Luthor. Terminus had
-never physically met anyone from LoD. But that scarcely mattered much--
-the Atlanta Three themselves had never been officially vetted by Lex, either.
-
-As far as law enforcement was concerned, the issues were clear.
-Terminus was a full-time, adult computer professional
-with particular skills at AT&T software and hardware--
-but Terminus reeked of the Legion of Doom and the underground.
-
-On February 1, 1990--half a month after the Martin Luther King Day Crash--
-USSS agents Tim Foley from Chicago, and Jack Lewis from the Baltimore office,
-accompanied by AT&T security officer Jerry Dalton, travelled to Middle Town,
-Maryland. There they grilled Terminus in his home (to the stark terror of
-his wife and small children), and, in their customary fashion, hauled his
-computers out the door.
-
-The Netsys machine proved to contain a plethora of arcane UNIX software--
-proprietary source code formally owned by AT&T. Software such as:
-UNIX System Five Release 3.2; UNIX SV Release 3.1; UUCP communications
-software; KORN SHELL; RFS; IWB; WWB; DWB; the C++ programming language;
-PMON; TOOL CHEST; QUEST; DACT, and S FIND.
-
-In the long-established piratical tradition of the underground,
-Terminus had been trading this illicitly-copied software with
-a small circle of fellow UNIX programmers. Very unwisely,
-he had stored seven years of his electronic mail on his Netsys machine,
-which documented all the friendly arrangements he had made with
-his various colleagues.
-
-Terminus had not crashed the AT&T phone system on January 15.
-He was, however, blithely running a not-for-profit AT&T
-software-piracy ring. This was not an activity AT&T found amusing.
-AT&T security officer Jerry Dalton valued this "stolen" property
-at over three hundred thousand dollars.
-
-AT&T's entry into the tussle of free enterprise had been complicated
-by the new, vague groundrules of the information economy.
-Until the break-up of Ma Bell, AT&T was forbidden to sell
-computer hardware or software. Ma Bell was the phone company;
-Ma Bell was not allowed to use the enormous revenue from
-telephone utilities, in order to finance any entry into
-the computer market.
-
-AT&T nevertheless invented the UNIX operating system.
-And somehow AT&T managed to make UNIX a minor source of income.
-Weirdly, UNIX was not sold as computer software,
-but actually retailed under an obscure regulatory
-exemption allowing sales of surplus equipment and scrap.
-Any bolder attempt to promote or retail UNIX would have
-aroused angry legal opposition from computer companies.
-Instead, UNIX was licensed to universities, at modest rates,
-where the acids of academic freedom ate away steadily at AT&T's
-proprietary rights.
-
-Come the breakup, AT&T recognized that UNIX was a potential gold-mine.
-By now, large chunks of UNIX code had been created that were not AT&T's,
-and were being sold by others. An entire rival UNIX-based operating system
-had arisen in Berkeley, California (one of the world's great founts of
-ideological hackerdom). Today, "hackers" commonly consider "Berkeley UNIX"
-to be technically superior to AT&T's "System V UNIX," but AT&T has not
-allowed mere technical elegance to intrude on the real-world business
-of marketing proprietary software. AT&T has made its own code deliberately
-incompatible with other folks' UNIX, and has written code that it can prove
-is copyrightable, even if that code happens to be somewhat awkward--"kludgey."
-AT&T UNIX user licenses are serious business agreements, replete with very
-clear copyright statements and non-disclosure clauses.
-
-AT&T has not exactly kept the UNIX cat in the bag,
-but it kept a grip on its scruff with some success.
-By the rampant, explosive standards of software piracy,
-AT&T UNIX source code is heavily copyrighted, well-guarded,
-well-licensed. UNIX was traditionally run only on
-mainframe machines, owned by large groups of suit-and-tie
-professionals, rather than on bedroom machines where
-people can get up to easy mischief.
-
-And AT&T UNIX source code is serious high-level programming.
-The number of skilled UNIX programmers with any actual motive
-to swipe UNIX source code is small. It's tiny, compared to
-the tens of thousands prepared to rip-off, say, entertaining
-PC games like "Leisure Suit Larry."
-
-But by 1989, the warez-d00d underground, in the persons of Terminus
-and his friends, was gnawing at AT&T UNIX. And the property in question
-was not sold for twenty bucks over the counter at the local branch of
-Babbage's or Egghead's; this was massive, sophisticated, multi-line,
-multi-author corporate code worth tens of thousands of dollars.
-
-It must be recognized at this point that Terminus's purported ring of UNIX
-software pirates had not actually made any money from their suspected crimes.
-The $300,000 dollar figure bandied about for the contents of Terminus's
-computer did not mean that Terminus was in actual illicit possession
-of three hundred thousand of AT&T's dollars. Terminus was shipping
-software back and forth, privately, person to person, for free.
-He was not making a commercial business of piracy. He hadn't
-asked for money; he didn't take money. He lived quite modestly.
-
-AT&T employees--as well as freelance UNIX consultants, like Terminus--
-commonly worked with "proprietary" AT&T software, both in the office
-and at home on their private machines. AT&T rarely sent security officers
-out to comb the hard disks of its consultants. Cheap freelance UNIX
-contractors were quite useful to AT&T; they didn't have health insurance
-or retirement programs, much less union membership in the Communication
-Workers of America. They were humble digital drudges, wandering with mop
-and bucket through the Great Technological Temple of AT&T; but when the
-Secret Service arrived at their homes, it seemed they were eating with
-company silverware and sleeping on company sheets! Outrageously, they
-behaved as if the things they worked with every day belonged to them!
-
-And these were no mere hacker teenagers with their hands full
-of trash-paper and their noses pressed to the corporate windowpane.
-These guys were UNIX wizards, not only carrying AT&T data in their
-machines and their heads, but eagerly networking about it,
-over machines that were far more powerful than anything previously
-imagined in private hands. How do you keep people disposable,
-yet assure their awestruck respect for your property? It was a dilemma.
-
-Much UNIX code was public-domain, available for free. Much "proprietary"
-UNIX code had been extensively re-written, perhaps altered so much that it
-became an entirely new product--or perhaps not. Intellectual property rights
-for software developers were, and are, extraordinarily complex and confused.
-And software "piracy," like the private copying of videos, is one of the most
-widely practiced "crimes" in the world today.
-
-The USSS were not experts in UNIX or familiar with the customs of its use.
-The United States Secret Service, considered as a body, did not have one single
-person in it who could program in a UNIX environment--no, not even one.
-The Secret Service WERE making extensive use of expert help, but the "experts"
-they had chosen were AT&T and Bellcore security officials, the very victims of
-the purported crimes under investigation, the very people whose interest in
-AT&T's "proprietary" software was most pronounced.
-
-On February 6, 1990, Terminus was arrested by Agent Lewis.
-Eventually, Terminus would be sent to prison for his illicit
-use of a piece of AT&T software.
-
-The issue of pirated AT&T software would bubble along in the background
-during the war on the Legion of Doom. Some half-dozen of Terminus's on-line
-acquaintances, including people in Illinois, Texas and California,
-were grilled by the Secret Service in connection with the illicit
-copying of software. Except for Terminus, however, none were charged
-with a crime. None of them shared his peculiar prominence in the
-hacker underground.
-
-But that did not mean that these people would, or could,
-stay out of trouble. The transferral of illicit data in
-cyberspace is hazy and ill-defined business, with paradoxical
-dangers for everyone concerned: hackers, signal carriers,
-board owners, cops, prosecutors, even random passers-by.
-Sometimes, well-meant attempts to avert trouble
-or punish wrongdoing bring more trouble than
-would simple ignorance, indifference or impropriety.
-
-Terminus's "Netsys" board was not a common-or-garden
-bulletin board system, though it had most of the usual
-functions of a board. Netsys was not a stand-alone machine,
-but part of the globe-spanning "UUCP" cooperative network.
-The UUCP network uses a set of Unix software programs called
-"Unix-to-Unix Copy," which allows Unix systems to throw data to
-one another at high speed through the public telephone network.
-UUCP is a radically decentralized, not-for-profit network of UNIX computers.
-There are tens of thousands of these UNIX machines. Some are small,
-but many are powerful and also link to other networks. UUCP has
-certain arcane links to major networks such as JANET, EasyNet, BITNET,
-JUNET, VNET, DASnet, PeaceNet and FidoNet, as well as the gigantic Internet.
-(The so-called "Internet" is not actually a network itself, but rather an
-"internetwork" connections standard that allows several globe-spanning
-computer networks to communicate with one another. Readers fascinated
-by the weird and intricate tangles of modern computer networks may enjoy
-John S. Quarterman's authoritative 719-page explication, The Matrix,
-Digital Press, 1990.)
-
-A skilled user of Terminus' UNIX machine could send and receive
-electronic mail from almost any major computer network in the world.
-Netsys was not called a "board" per se, but rather a "node."
-"Nodes" were larger, faster, and more sophisticated than mere "boards,"
-and for hackers, to hang out on internationally-connected "nodes"
-was quite the step up from merely hanging out on local "boards."
-
-Terminus's Netsys node in Maryland had a number of direct
-links to other, similar UUCP nodes, run by people who shared his
-interests and at least something of his free-wheeling attitude.
-One of these nodes was Jolnet, owned by Richard Andrews, who,
-like Terminus, was an independent UNIX consultant.
-Jolnet also ran UNIX, and could be contacted at high speed
-by mainframe machines from all over the world. Jolnet was
-quite a sophisticated piece of work, technically speaking,
-but it was still run by an individual, as a private,
-not-for-profit hobby. Jolnet was mostly used by other
-UNIX programmers--for mail, storage, and access to networks.
-Jolnet supplied access network access to about two hundred people,
-as well as a local junior college.
-
-Among its various features and services, Jolnet also carried
-Phrack magazine.
-
-For reasons of his own, Richard Andrews had become suspicious
-of a new user called "Robert Johnson." Richard Andrews
-took it upon himself to have a look at what "Robert Johnson"
-was storing in Jolnet. And Andrews found the E911 Document.
-
-"Robert Johnson" was the Prophet from the Legion of Doom,
-and the E911 Document was illicitly copied data from Prophet's
-raid on the BellSouth computers.
-
-The E911 Document, a particularly illicit piece of digital property,
-was about to resume its long, complex, and disastrous career.
-
-It struck Andrews as fishy that someone not a telephone employee
-should have a document referring to the "Enhanced 911 System."
-Besides, the document itself bore an obvious warning.
-
-"WARNING: NOT FOR USE OR DISCLOSURE OUTSIDE BELLSOUTH
-OR ANY OF ITS SUBSIDIARIES EXCEPT UNDER WRITTEN AGREEMENT."
-
-These standard nondisclosure tags are often appended to all sorts
-of corporate material. Telcos as a species are particularly notorious
-for stamping most everything in sight as "not for use or disclosure."
-Still, this particular piece of data was about the 911 System.
-That sounded bad to Rich Andrews.
-
-Andrews was not prepared to ignore this sort of trouble.
-He thought it would be wise to pass the document along
-to a friend and acquaintance on the UNIX network, for consultation.
-So, around September 1988, Andrews sent yet another copy of the
-E911 Document electronically to an AT&T employee, one Charles Boykin,
-who ran a UNIX-based node called "attctc" in Dallas, Texas.
-
-"Attctc" was the property of AT&T, and was run from AT&T's
-Customer Technology Center in Dallas, hence the name "attctc."
-"Attctc" was better-known as "Killer," the name of the machine
-that the system was running on. "Killer" was a hefty, powerful,
-AT&T 3B2 500 model, a multi-user, multi-tasking UNIX platform
-with 32 meg of memory and a mind-boggling 3.2 Gigabytes of storage.
-When Killer had first arrived in Texas, in 1985, the 3B2 had been
-one of AT&T's great white hopes for going head-to-head with IBM
-for the corporate computer-hardware market. "Killer" had been shipped
-to the Customer Technology Center in the Dallas Infomart, essentially
-a high-technology mall, and there it sat, a demonstration model.
-
-Charles Boykin, a veteran AT&T hardware and digital communications expert,
-was a local technical backup man for the AT&T 3B2 system. As a display model
-in the Infomart mall, "Killer" had little to do, and it seemed a shame
-to waste the system's capacity. So Boykin ingeniously wrote some UNIX
-bulletin-board software for "Killer," and plugged the machine in to the
-local phone network. "Killer's" debut in late 1985 made it the first
-publicly available UNIX site in the state of Texas. Anyone who wanted to
-play was welcome.
-
-The machine immediately attracted an electronic community.
-It joined the UUCP network, and offered network links
-to over eighty other computer sites, all of which became dependent
-on Killer for their links to the greater world of cyberspace.
-And it wasn't just for the big guys; personal computer users
-also stored freeware programs for the Amiga, the Apple,
-the IBM and the Macintosh on Killer's vast 3,200 meg archives.
-At one time, Killer had the largest library of public-domain
-Macintosh software in Texas.
-
-Eventually, Killer attracted about 1,500 users,
-all busily communicating, uploading and downloading,
-getting mail, gossipping, and linking to arcane
-and distant networks.
-
-Boykin received no pay for running Killer. He considered
-it good publicity for the AT&T 3B2 system (whose sales were
-somewhat less than stellar), but he also simply enjoyed
-the vibrant community his skill had created. He gave away
-the bulletin-board UNIX software he had written, free of charge.
-
-In the UNIX programming community, Charlie Boykin had the
-reputation of a warm, open-hearted, level-headed kind of guy.
-In 1989, a group of Texan UNIX professionals voted Boykin
-"System Administrator of the Year." He was considered
-a fellow you could trust for good advice.
-
-In September 1988, without warning, the E911 Document
-came plunging into Boykin's life, forwarded by Richard Andrews.
-Boykin immediately recognized that the Document was hot property.
-He was not a voice-communications man, and knew little about
-the ins and outs of the Baby Bells, but he certainly knew what
-the 911 System was, and he was angry to see confidential data
-about it in the hands of a nogoodnik. This was clearly a
-matter for telco security. So, on September 21, 1988, Boykin
-made yet ANOTHER copy of the E911 Document and passed this
-one along to a professional acquaintance of his, one Jerome Dalton,
-from AT&T Corporate Information Security. Jerry Dalton was the
-very fellow who would later raid Terminus's house.
-
-From AT&T's security division, the E911 Document went to Bellcore.
-
-Bellcore (or BELL COmmunications REsearch) had once been the central
-laboratory of the Bell System. Bell Labs employees had invented
-the UNIX operating system. Now Bellcore was a quasi-independent,
-jointly owned company that acted as the research arm for all seven
-of the Baby Bell RBOCs. Bellcore was in a good position to co-ordinate
-security technology and consultation for the RBOCs, and the gentleman in
-charge of this effort was Henry M. Kluepfel, a veteran of the Bell System
-who had worked there for twenty-four years.
-
-On October 13, 1988, Dalton passed the E911 Document to Henry Kluepfel.
-Kluepfel, a veteran expert witness in telecommunications fraud and
-computer-fraud cases, had certainly seen worse trouble than this.
-He recognized the document for what it was: a trophy from a hacker break-in.
-
-However, whatever harm had been done in the intrusion was presumably old news.
-At this point there seemed little to be done. Kluepfel made a careful note
-of the circumstances and shelved the problem for the time being.
-
-Whole months passed.
-
-February 1989 arrived. The Atlanta Three were living it up
-in Bell South's switches, and had not yet met their comeuppance.
-The Legion was thriving. So was Phrack magazine.
-A good six months had passed since Prophet's AIMSX break-in.
-Prophet, as hackers will, grew weary of sitting on his laurels.
-"Knight Lightning" and "Taran King," the editors of Phrack,
-were always begging Prophet for material they could publish.
-Prophet decided that the heat must be off by this time,
-and that he could safely brag, boast, and strut.
-
-So he sent a copy of the E911 Document--yet another one--
-from Rich Andrews' Jolnet machine to Knight Lightning's
-BITnet account at the University of Missouri.
-Let's review the fate of the document so far.
-
-0. The original E911 Document. This in the AIMSX system
-on a mainframe computer in Atlanta, available to hundreds of people,
-but all of them, presumably, BellSouth employees. An unknown number
-of them may have their own copies of this document, but they are all
-professionals and all trusted by the phone company.
-
-1. Prophet's illicit copy, at home on his own computer in Decatur, Georgia.
-
-2. Prophet's back-up copy, stored on Rich Andrew's Jolnet machine
- in the basement of Rich Andrews' house near Joliet Illinois.
-
-3. Charles Boykin's copy on "Killer" in Dallas, Texas,
- sent by Rich Andrews from Joliet.
-
-4. Jerry Dalton's copy at AT&T Corporate Information Security in New Jersey,
- sent from Charles Boykin in Dallas.
-
-5. Henry Kluepfel's copy at Bellcore security headquarters in New Jersey,
- sent by Dalton.
-6. Knight Lightning's copy, sent by Prophet from Rich Andrews' machine,
- and now in Columbia, Missouri.
-
-We can see that the "security" situation of this proprietary document,
-once dug out of AIMSX, swiftly became bizarre. Without any money
-changing hands, without any particular special effort, this data
-had been reproduced at least six times and had spread itself all over
-the continent. By far the worst, however, was yet to come.
-
-In February 1989, Prophet and Knight Lightning bargained electronically
-over the fate of this trophy. Prophet wanted to boast, but, at the same time,
-scarcely wanted to be caught.
-
-For his part, Knight Lightning was eager to publish as much of the document
-as he could manage. Knight Lightning was a fledgling political-science major
-with a particular interest in freedom-of-information issues. He would gladly
-publish most anything that would reflect glory on the prowess of the
-underground and embarrass the telcos. However, Knight Lightning himself
-had contacts in telco security, and sometimes consulted them on material
-he'd received that might be too dicey for publication.
-
-Prophet and Knight Lightning decided to edit the E911 Document
-so as to delete most of its identifying traits. First of all,
-its large "NOT FOR USE OR DISCLOSURE" warning had to go.
-Then there were other matters. For instance, it listed
-the office telephone numbers of several BellSouth 911
-specialists in Florida. If these phone numbers were
-published in Phrack, the BellSouth employees involved
-would very likely be hassled by phone phreaks,
-which would anger BellSouth no end, and pose a
-definite operational hazard for both Prophet and Phrack.
-
-So Knight Lightning cut the Document almost in half,
-removing the phone numbers and some of the touchier
-and more specific information. He passed it back
-electronically to Prophet; Prophet was still nervous,
-so Knight Lightning cut a bit more. They finally agreed
-that it was ready to go, and that it would be published
-in Phrack under the pseudonym, "The Eavesdropper."
-
-And this was done on February 25, 1989.
-
-The twenty-fourth issue of Phrack featured a chatty interview
-with co-ed phone-phreak "Chanda Leir," three articles on BITNET
-and its links to other computer networks, an article on 800 and 900
-numbers by "Unknown User," "VaxCat's" article on telco basics
-(slyly entitled "Lifting Ma Bell's Veil of Secrecy,)" and
-the usual "Phrack World News."
-
-The News section, with painful irony, featured an extended account
-of the sentencing of "Shadowhawk," an eighteen-year-old Chicago hacker
-who had just been put in federal prison by William J. Cook himself.
-
-And then there were the two articles by "The Eavesdropper."
-The first was the edited E911 Document, now titled
-"Control Office Administration Of Enhanced 911 Services
-for Special Services and Major Account Centers."
-Eavesdropper's second article was a glossary of terms
-explaining the blizzard of telco acronyms and buzzwords
-in the E911 Document.
-
-The hapless document was now distributed, in the usual Phrack routine,
-to a good one hundred and fifty sites. Not a hundred and fifty PEOPLE,
-mind you--a hundred and fifty SITES, some of these sites linked to UNIX
-nodes or bulletin board systems, which themselves had readerships of tens,
-dozens, even hundreds of people.
-
-This was February 1989. Nothing happened immediately.
-Summer came, and the Atlanta crew were raided by the Secret Service.
-Fry Guy was apprehended. Still nothing whatever happened to Phrack.
-Six more issues of Phrack came out, 30 in all, more or less on
-a monthly schedule. Knight Lightning and co-editor Taran King
-went untouched.
-
-Phrack tended to duck and cover whenever the heat came down.
-During the summer busts of 1987--(hacker busts tended to cluster in summer,
-perhaps because hackers were easier to find at home than in college)--
-Phrack had ceased publication for several months, and laid low.
-Several LoD hangers-on had been arrested, but nothing had happened
-to the Phrack crew, the premiere gossips of the underground.
-In 1988, Phrack had been taken over by a new editor,
-"Crimson Death," a raucous youngster with a taste for anarchy files.
-1989, however, looked like a bounty year for the underground.
-Knight Lightning and his co-editor Taran King took up the reins again,
-and Phrack flourished throughout 1989. Atlanta LoD went down hard in
-the summer of 1989, but Phrack rolled merrily on. Prophet's E911 Document
-seemed unlikely to cause Phrack any trouble. By January 1990,
-it had been available in Phrack for almost a year. Kluepfel and Dalton,
-officers of Bellcore and AT&T security, had possessed the document
-for sixteen months--in fact, they'd had it even before Knight Lightning
-himself, and had done nothing in particular to stop its distribution.
-They hadn't even told Rich Andrews or Charles Boykin to erase the copies
-from their UNIX nodes, Jolnet and Killer.
-
-But then came the monster Martin Luther King Day Crash of January 15, 1990.
-
-A flat three days later, on January 18, four agents showed up
-at Knight Lightning's fraternity house. One was Timothy Foley,
-the second Barbara Golden, both of them Secret Service agents
-from the Chicago office. Also along was a University of Missouri
-security officer, and Reed Newlin, a security man from Southwestern Bell,
-the RBOC having jurisdiction over Missouri.
-
-Foley accused Knight Lightning of causing the nationwide crash
-of the phone system.
-
-Knight Lightning was aghast at this allegation. On the face of it,
-the suspicion was not entirely implausible--though Knight Lightning
-knew that he himself hadn't done it. Plenty of hot-dog hackers
-had bragged that they could crash the phone system, however.
-"Shadowhawk," for instance, the Chicago hacker whom William Cook
-had recently put in jail, had several times boasted on boards
-that he could "shut down AT&T's public switched network."
-
-And now this event, or something that looked just like it,
-had actually taken place. The Crash had lit a fire under
-the Chicago Task Force. And the former fence-sitters at
-Bellcore and AT&T were now ready to roll. The consensus
-among telco security--already horrified by the skill of
-the BellSouth intruders --was that the digital underground
-was out of hand. LoD and Phrack must go. And in publishing
-Prophet's E911 Document, Phrack had provided law enforcement
-with what appeared to be a powerful legal weapon.
-
-Foley confronted Knight Lightning about the E911 Document.
-
-Knight Lightning was cowed. He immediately began "cooperating fully"
-in the usual tradition of the digital underground.
-
-He gave Foley a complete run of Phrack, printed out in a set
-of three-ring binders. He handed over his electronic mailing list
-of Phrack subscribers. Knight Lightning was grilled for four hours
-by Foley and his cohorts. Knight Lightning admitted that Prophet
-had passed him the E911 Document, and he admitted that he had known
-it was stolen booty from a hacker raid on a telephone company.
-Knight Lightning signed a statement to this effect, and agreed,
-in writing, to cooperate with investigators.
-
-Next day--January 19, 1990, a Friday --the Secret Service returned
-with a search warrant, and thoroughly searched Knight Lightning's
-upstairs room in the fraternity house. They took all his floppy disks,
-though, interestingly, they left Knight Lightning in possession
-of both his computer and his modem. (The computer had no hard disk,
-and in Foley's judgement was not a store of evidence.) But this was a
-very minor bright spot among Knight Lightning's rapidly multiplying troubles.
-By this time, Knight Lightning was in plenty of hot water, not only with
-federal police, prosecutors, telco investigators, and university security,
-but with the elders of his own campus fraternity, who were outraged
-to think that they had been unwittingly harboring a federal computer-criminal.
-
-On Monday, Knight Lightning was summoned to Chicago, where he was
-further grilled by Foley and USSS veteran agent Barbara Golden, this time
-with an attorney present. And on Tuesday, he was formally indicted
-by a federal grand jury.
-
-The trial of Knight Lightning, which occurred on July 24-27, 1990,
-was the crucial show-trial of the Hacker Crackdown. We will examine
-the trial at some length in Part Four of this book.
-
-In the meantime, we must continue our dogged pursuit of the E911 Document.
-
-It must have been clear by January 1990 that the E911 Document,
-in the form Phrack had published it back in February 1989,
-had gone off at the speed of light in at least a hundred
-and fifty different directions. To attempt to put this
-electronic genie back in the bottle was flatly impossible.
-
-And yet, the E911 Document was STILL stolen property,
-formally and legally speaking. Any electronic transference
-of this document, by anyone unauthorized to have it,
-could be interpreted as an act of wire fraud. Interstate
-transfer of stolen property, including electronic property,
-was a federal crime.
-
-The Chicago Computer Fraud and Abuse Task Force had been assured
-that the E911 Document was worth a hefty sum of money. In fact,
-they had a precise estimate of its worth from BellSouth security personnel:
-$79,449. A sum of this scale seemed to warrant vigorous prosecution.
-Even if the damage could not be undone, at least this large sum
-offered a good legal pretext for stern punishment of the thieves.
-It seemed likely to impress judges and juries. And it could be used
-in court to mop up the Legion of Doom.
-
-The Atlanta crowd was already in the bag, by the time
-the Chicago Task Force had gotten around to Phrack.
-But the Legion was a hydra-headed thing. In late 89,
-a brand-new Legion of Doom board, "Phoenix Project,"
-had gone up in Austin, Texas. Phoenix Project was sysoped
-by no less a man than the Mentor himself, ably assisted by
-University of Texas student and hardened Doomster "Erik Bloodaxe."
-
-As we have seen from his Phrack manifesto, the Mentor was a hacker
-zealot who regarded computer intrusion as something close to a moral duty.
-Phoenix Project was an ambitious effort, intended to revive the digital
-underground to what Mentor considered the full flower of the early 80s.
-The Phoenix board would also boldly bring elite hackers face-to-face
-with the telco "opposition." On "Phoenix," America's cleverest hackers
-would supposedly shame the telco squareheads out of their stick-in-the-mud
-attitudes, and perhaps convince them that the Legion of Doom elite were really
-an all-right crew. The premiere of "Phoenix Project" was heavily trumpeted
-by Phrack,and "Phoenix Project" carried a complete run of Phrack issues,
-including the E911 Document as Phrack had published it.
-
-Phoenix Project was only one of many--possibly hundreds--of nodes and boards
-all over America that were in guilty possession of the E911 Document.
-But Phoenix was an outright, unashamed Legion of Doom board.
-Under Mentor's guidance, it was flaunting itself in the face
-of telco security personnel. Worse yet, it was actively trying
-to WIN THEM OVER as sympathizers for the digital underground elite.
-"Phoenix" had no cards or codes on it. Its hacker elite considered
-Phoenix at least technically legal. But Phoenix was a corrupting influence,
-where hacker anarchy was eating away like digital acid at the underbelly
-of corporate propriety.
-
-The Chicago Computer Fraud and Abuse Task Force now prepared
-to descend upon Austin, Texas.
-
-Oddly, not one but TWO trails of the Task Force's investigation led
-toward Austin. The city of Austin, like Atlanta, had made itself
-a bulwark of the Sunbelt's Information Age, with a strong university
-research presence, and a number of cutting-edge electronics companies,
-including Motorola, Dell, CompuAdd, IBM, Sematech and MCC.
-
-Where computing machinery went, hackers generally followed.
-Austin boasted not only "Phoenix Project," currently LoD's
-most flagrant underground board, but a number of UNIX nodes.
-
-One of these nodes was "Elephant," run by a UNIX consultant
-named Robert Izenberg. Izenberg, in search of a relaxed Southern
-lifestyle and a lowered cost-of-living, had recently migrated
-to Austin from New Jersey. In New Jersey, Izenberg had worked
-for an independent contracting company, programming UNIX code for
-AT&T itself. "Terminus" had been a frequent user on Izenberg's
-privately owned Elephant node.
-
-Having interviewed Terminus and examined the records on Netsys,
-the Chicago Task Force were now convinced that they had discovered
-an underground gang of UNIX software pirates, who were demonstrably
-guilty of interstate trafficking in illicitly copied AT&T source code.
-Izenberg was swept into the dragnet around Terminus, the self-proclaimed
-ultimate UNIX hacker.
-
-Izenberg, in Austin, had settled down into a UNIX job
-with a Texan branch of IBM. Izenberg was no longer
-working as a contractor for AT&T, but he had friends
-in New Jersey, and he still logged on to AT&T UNIX
-computers back in New Jersey, more or less whenever
-it pleased him. Izenberg's activities appeared highly
-suspicious to the Task Force. Izenberg might well be
-breaking into AT&T computers, swiping AT&T software,
-and passing it to Terminus and other possible confederates,
-through the UNIX node network. And this data was worth,
-not merely $79,499, but hundreds of thousands of dollars!
-
-On February 21, 1990, Robert Izenberg arrived home
-from work at IBM to find that all the computers
-had mysteriously vanished from his Austin apartment.
-Naturally he assumed that he had been robbed.
-His "Elephant" node, his other machines, his notebooks,
-his disks, his tapes, all gone! However, nothing much
-else seemed disturbed--the place had not been ransacked.
-The puzzle becaming much stranger some five minutes later.
-Austin U. S. Secret Service Agent Al Soliz, accompanied by
-University of Texas campus-security officer Larry Coutorie
-and the ubiquitous Tim Foley, made their appearance at Izenberg's door.
-They were in plain clothes: slacks, polo shirts. They came in,
-and Tim Foley accused Izenberg of belonging to the Legion of Doom.
-
-Izenberg told them that he had never heard of the "Legion of Doom."
-And what about a certain stolen E911 Document, that posed a direct
-threat to the police emergency lines? Izenberg claimed that he'd
-never heard of that, either.
-
-His interrogators found this difficult to believe.
-Didn't he know Terminus?
-
-Who?
-
-They gave him Terminus's real name. Oh yes, said Izenberg.
-He knew THAT guy all right--he was leading discussions
-on the Internet about AT&T computers, especially the AT&T 3B2.
-
-AT&T had thrust this machine into the marketplace,
-but, like many of AT&T's ambitious attempts to enter
-the computing arena, the 3B2 project had something less
-than a glittering success. Izenberg himself had been
-a contractor for the division of AT&T that supported the 3B2.
-The entire division had been shut down.
-
-Nowadays, the cheapest and quickest way to get help with this
-fractious piece of machinery was to join one of Terminus's
-discussion groups on the Internet, where friendly and knowledgeable
-hackers would help you for free. Naturally the remarks within this
-group were less than flattering about the Death Star. . .was
-THAT the problem?
-
-Foley told Izenberg that Terminus had been acquiring hot software
-through his, Izenberg's, machine.
-
-Izenberg shrugged this off. A good eight megabytes of data flowed
-through his UUCP site every day. UUCP nodes spewed data like fire hoses.
-Elephant had been directly linked to Netsys--not surprising, since Terminus
-was a 3B2 expert and Izenberg had been a 3B2 contractor.
-Izenberg was also linked to "attctc" and the University of Texas.
-Terminus was a well-known UNIX expert, and might have been up to
-all manner of hijinks on Elephant. Nothing Izenberg could do about that.
-That was physically impossible. Needle in a haystack.
-
-In a four-hour grilling, Foley urged Izenberg to come clean
-and admit that he was in conspiracy with Terminus,
-and a member of the Legion of Doom.
-
-Izenberg denied this. He was no weirdo teenage hacker--
-he was thirty-two years old, and didn't even have a "handle."
-Izenberg was a former TV technician and electronics specialist
-who had drifted into UNIX consulting as a full-grown adult.
-Izenberg had never met Terminus, physically. He'd once bought
-a cheap high-speed modem from him, though.
-
-Foley told him that this modem (a Telenet T2500 which ran at 19.2 kilobaud,
-and which had just gone out Izenberg's door in Secret Service custody)
-was likely hot property. Izenberg was taken aback to hear this; but then
-again, most of Izenberg's equipment, like that of most freelance professionals
-in the industry, was discounted, passed hand-to-hand through various kinds
-of barter and gray-market. There was no proof that the modem was stolen,
-and even if it were, Izenberg hardly saw how that gave them the right
-to take every electronic item in his house.
-
-Still, if the United States Secret Service figured they needed
-his computer for national security reasons--or whatever--
-then Izenberg would not kick. He figured he would somehow
-make the sacrifice of his twenty thousand dollars' worth
-of professional equipment, in the spirit of full cooperation
-and good citizenship.
-
-Robert Izenberg was not arrested. Izenberg was not charged with any crime.
-His UUCP node--full of some 140 megabytes of the files, mail, and data
-of himself and his dozen or so entirely innocent users--went out the door
-as "evidence." Along with the disks and tapes, Izenberg had lost about
-800 megabytes of data.
-
-Six months would pass before Izenberg decided to phone the Secret Service
-and ask how the case was going. That was the first time that Robert Izenberg
-would ever hear the name of William Cook. As of January 1992, a full
-two years after the seizure, Izenberg, still not charged with any crime,
-would be struggling through the morass of the courts, in hope of recovering
-his thousands of dollars' worth of seized equipment.
-
-In the meantime, the Izenberg case received absolutely no press coverage.
-The Secret Service had walked into an Austin home, removed a UNIX bulletin-
-board system, and met with no operational difficulties whatsoever.
-
-Except that word of a crackdown had percolated through the Legion of Doom.
-"The Mentor" voluntarily shut down "The Phoenix Project." It seemed a pity,
-especially as telco security employees had, in fact, shown up on Phoenix,
-just as he had hoped--along with the usual motley crowd of LoD heavies,
-hangers-on, phreaks, hackers and wannabes. There was "Sandy" Sandquist from
-US SPRINT security, and some guy named Henry Kluepfel, from Bellcore itself!
-Kluepfel had been trading friendly banter with hackers on Phoenix since
-January 30th (two weeks after the Martin Luther King Day Crash).
-The presence of such a stellar telco official seemed quite the coup
-for Phoenix Project.
-
-Still, Mentor could judge the climate. Atlanta in ruins,
-Phrack in deep trouble, something weird going on with UNIX nodes--
-discretion was advisable. Phoenix Project went off-line.
-
-Kluepfel, of course, had been monitoring this LoD bulletin
-board for his own purposes--and those of the Chicago unit.
-As far back as June 1987, Kluepfel had logged on to a Texas
-underground board called "Phreak Klass 2600." There he'd
-discovered an Chicago youngster named "Shadowhawk,"
-strutting and boasting about rifling AT&T computer files,
-and bragging of his ambitions to riddle AT&T's Bellcore
-computers with trojan horse programs. Kluepfel had passed
-the news to Cook in Chicago, Shadowhawk's computers
-had gone out the door in Secret Service custody,
-and Shadowhawk himself had gone to jail.
-
-Now it was Phoenix Project's turn. Phoenix Project postured
-about "legality" and "merely intellectual interest," but it reeked
-of the underground. It had Phrack on it. It had the E911 Document.
-It had a lot of dicey talk about breaking into systems, including some
-bold and reckless stuff about a supposed "decryption service" that Mentor
-and friends were planning to run, to help crack encrypted passwords off
-of hacked systems.
-
-Mentor was an adult. There was a bulletin board at his place of work,
-as well. Kleupfel logged onto this board, too, and discovered it to be
-called "Illuminati." It was run by some company called Steve Jackson Games.
-
-On March 1, 1990, the Austin crackdown went into high gear.
-
-On the morning of March 1--a Thursday--21-year-old University of Texas
-student "Erik Bloodaxe," co-sysop of Phoenix Project and an avowed member
-of the Legion of Doom, was wakened by a police revolver levelled at his head.
-
-Bloodaxe watched, jittery, as Secret Service agents
-appropriated his 300 baud terminal and, rifling his files,
-discovered his treasured source-code for Robert Morris's
-notorious Internet Worm. But Bloodaxe, a wily operator,
-had suspected that something of the like might be coming.
-All his best equipment had been hidden away elsewhere.
-The raiders took everything electronic, however,
-including his telephone. They were stymied by his
-hefty arcade-style Pac-Man game, and left it in place,
-as it was simply too heavy to move.
-
-Bloodaxe was not arrested. He was not charged with any crime.
-A good two years later, the police still had what they had
-taken from him, however.
-
-The Mentor was less wary. The dawn raid rousted him and his wife
-from bed in their underwear, and six Secret Service agents,
-accompanied by an Austin policeman and Henry Kluepfel himself,
-made a rich haul. Off went the works, into the agents' white
-Chevrolet minivan: an IBM PC-AT clone with 4 meg of RAM and
-a 120-meg hard disk; a Hewlett-Packard LaserJet II printer;
-a completely legitimate and highly expensive SCO-Xenix 286
-operating system; Pagemaker disks and documentation;
-and the Microsoft Word word-processing program. Mentor's wife
-had her incomplete academic thesis stored on the hard-disk;
-that went, too, and so did the couple's telephone. As of two years later,
-all this property remained in police custody.
-
-Mentor remained under guard in his apartment as agents prepared
-to raid Steve Jackson Games. The fact that this was a business
-headquarters and not a private residence did not deter the agents.
-It was still very early; no one was at work yet. The agents prepared
-to break down the door, but Mentor, eavesdropping on the Secret Service
-walkie-talkie traffic, begged them not to do it, and offered his key
-to the building.
-
-The exact details of the next events are unclear. The agents
-would not let anyone else into the building. Their search warrant,
-when produced, was unsigned. Apparently they breakfasted from the local
-"Whataburger," as the litter from hamburgers was later found inside.
-They also extensively sampled a bag of jellybeans kept by an SJG employee.
-Someone tore a "Dukakis for President" sticker from the wall.
-
-SJG employees, diligently showing up for the day's work, were met
-at the door and briefly questioned by U.S. Secret Service agents.
-The employees watched in astonishment as agents wielding crowbars
-and screwdrivers emerged with captive machines. They attacked
-outdoor storage units with boltcutters. The agents wore
-blue nylon windbreakers with "SECRET SERVICE" stencilled
-across the back, with running-shoes and jeans.
-
-Jackson's company lost three computers, several hard-disks,
-hundred of floppy disks, two monitors, three modems,
-a laser printer, various powercords, cables, and adapters
-(and, oddly, a small bag of screws, bolts and nuts).
-The seizure of Illuminati BBS deprived SJG of all the programs,
-text files, and private e-mail on the board. The loss of two other
-SJG computers was a severe blow as well, since it caused the loss
-of electronically stored contracts, financial projections,
-address directories, mailing lists, personnel files,
-business correspondence, and, not least, the drafts
-of forthcoming games and gaming books.
-
-No one at Steve Jackson Games was arrested. No one was accused
-of any crime. No charges were filed. Everything appropriated
-was officially kept as "evidence" of crimes never specified.
-
-After the Phrack show-trial, the Steve Jackson Games scandal
-was the most bizarre and aggravating incident of the Hacker
-Crackdown of 1990. This raid by the Chicago Task Force
-on a science-fiction gaming publisher was to rouse a
-swarming host of civil liberties issues, and gave rise
-to an enduring controversy that was still re-complicating itself,
-and growing in the scope of its implications, a full two years later.
-
-The pursuit of the E911 Document stopped with the Steve Jackson Games raid.
-As we have seen, there were hundreds, perhaps thousands of computer users
-in America with the E911 Document in their possession. Theoretically,
-Chicago had a perfect legal right to raid any of these people,
-and could have legally seized the machines of anybody who subscribed to Phrack.
-However, there was no copy of the E911 Document on Jackson's Illuminati board.
-And there the Chicago raiders stopped dead; they have not raided anyone since.
-
-It might be assumed that Rich Andrews and Charlie Boykin, who had brought
-the E911 Document to the attention of telco security, might be spared
-any official suspicion. But as we have seen, the willingness to
-"cooperate fully" offers little, if any, assurance against federal
-anti-hacker prosecution.
-
-Richard Andrews found himself in deep trouble, thanks to the E911 Document.
-Andrews lived in Illinois, the native stomping grounds of the Chicago
-Task Force. On February 3 and 6, both his home and his place of work
-were raided by USSS. His machines went out the door, too, and he was
-grilled at length (though not arrested). Andrews proved to be in
-purportedly guilty possession of: UNIX SVR 3.2; UNIX SVR 3.1; UUCP;
-PMON; WWB; IWB; DWB; NROFF; KORN SHELL '88; C++; and QUEST,
-among other items. Andrews had received this proprietary code--
-which AT&T officially valued at well over $250,000--through the
-UNIX network, much of it supplied to him as a personal favor by Terminus.
-Perhaps worse yet, Andrews admitted to returning the favor, by passing
-Terminus a copy of AT&T proprietary STARLAN source code.
-
-Even Charles Boykin, himself an AT&T employee, entered some very hot water.
-By 1990, he'd almost forgotten about the E911 problem he'd reported in
-September 88; in fact, since that date, he'd passed two more security alerts
-to Jerry Dalton, concerning matters that Boykin considered far worse than
-the E911 Document.
-
-But by 1990, year of the crackdown, AT&T Corporate Information Security
-was fed up with "Killer." This machine offered no direct income to AT&T,
-and was providing aid and comfort to a cloud of suspicious yokels
-from outside the company, some of them actively malicious toward AT&T,
-its property, and its corporate interests. Whatever goodwill and publicity
-had been won among Killer's 1,500 devoted users was considered no longer
-worth the security risk. On February 20, 1990, Jerry Dalton arrived in
-Dallas and simply unplugged the phone jacks, to the puzzled alarm
-of Killer's many Texan users. Killer went permanently off-line,
-with the loss of vast archives of programs and huge quantities
-of electronic mail; it was never restored to service. AT&T showed
-no particular regard for the "property" of these 1,500 people.
-Whatever "property" the users had been storing on AT&T's computer
-simply vanished completely.
-
-Boykin, who had himself reported the E911 problem,
-now found himself under a cloud of suspicion. In a weird
-private-security replay of the Secret Service seizures,
-Boykin's own home was visited by AT&T Security and his
-own machines were carried out the door.
-
-However, there were marked special features in the Boykin case.
-Boykin's disks and his personal computers were swiftly examined
-by his corporate employers and returned politely in just two days--
-(unlike Secret Service seizures, which commonly take months or years).
-Boykin was not charged with any crime or wrongdoing, and he kept his job
-with AT&T (though he did retire from AT&T in September 1991,
-at the age of 52).
-
-It's interesting to note that the US Secret Service somehow failed
-to seize Boykin's "Killer" node and carry AT&T's own computer out the door.
-Nor did they raid Boykin's home. They seemed perfectly willing to take the
-word of AT&T Security that AT&T's employee, and AT&T's "Killer" node,
-were free of hacker contraband and on the up-and-up.
-
-It's digital water-under-the-bridge at this point, as Killer's
-3,200 megabytes of Texan electronic community were erased in 1990,
-and "Killer" itself was shipped out of the state.
-
-But the experiences of Andrews and Boykin, and the users of their systems,
-remained side issues. They did not begin to assume the social, political,
-and legal importance that gathered, slowly but inexorably, around the issue
-of the raid on Steve Jackson Games.
-
-#
-
-We must now turn our attention to Steve Jackson Games itself,
-and explain what SJG was, what it really did, and how it had
-managed to attract this particularly odd and virulent kind of trouble.
-The reader may recall that this is not the first but the second time
-that the company has appeared in this narrative; a Steve Jackson game
-called GURPS was a favorite pastime of Atlanta hacker Urvile,
-and Urvile's science-fictional gaming notes had been mixed up
-promiscuously with notes about his actual computer intrusions.
-
-First, Steve Jackson Games, Inc., was NOT a publisher of "computer games."
-SJG published "simulation games," parlor games that were played on paper,
-with pencils, and dice, and printed guidebooks full of rules and
-statistics tables. There were no computers involved in the games themselves.
-When you bought a Steve Jackson Game, you did not receive any software disks.
-What you got was a plastic bag with some cardboard game tokens,
-maybe a few maps or a deck of cards. Most of their products were books.
-
-However, computers WERE deeply involved in the Steve Jackson Games business.
-Like almost all modern publishers, Steve Jackson and his fifteen employees
-used computers to write text, to keep accounts, and to run the business
-generally. They also used a computer to run their official bulletin board
-system for Steve Jackson Games, a board called Illuminati. On Illuminati,
-simulation gamers who happened to own computers and modems could associate,
-trade mail, debate the theory and practice of gaming, and keep up with the
-company's news and its product announcements.
-
-Illuminati was a modestly popular board, run on a small computer
-with limited storage, only one phone-line, and no ties to large-scale
-computer networks. It did, however, have hundreds of users,
-many of them dedicated gamers willing to call from out-of-state.
-
-Illuminati was NOT an "underground" board. It did not feature hints
-on computer intrusion, or "anarchy files," or illicitly posted
-credit card numbers, or long-distance access codes.
-Some of Illuminati's users, however, were members of the Legion of Doom.
-And so was one of Steve Jackson's senior employees--the Mentor.
-The Mentor wrote for Phrack, and also ran an underground board,
-Phoenix Project--but the Mentor was not a computer professional.
-The Mentor was the managing editor of Steve Jackson Games and
-a professional game designer by trade. These LoD members did not
-use Illuminati to help their HACKING activities. They used it to
-help their GAME-PLAYING activities--and they were even more dedicated
-to simulation gaming than they were to hacking.
-
-"Illuminati" got its name from a card-game that Steve Jackson himself,
-the company's founder and sole owner, had invented. This multi-player
-card-game was one of Mr Jackson's best-known, most successful,
-most technically innovative products. "Illuminati" was a game
-of paranoiac conspiracy in which various antisocial cults warred
-covertly to dominate the world. "Illuminati" was hilarious,
-and great fun to play, involving flying saucers, the CIA, the KGB,
-the phone companies, the Ku Klux Klan, the South American Nazis,
-the cocaine cartels, the Boy Scouts, and dozens of other splinter groups
-from the twisted depths of Mr. Jackson's professionally fervid imagination.
-For the uninitiated, any public discussion of the "Illuminati" card-game
-sounded, by turns, utterly menacing or completely insane.
-
-And then there was SJG's "Car Wars," in which souped-up armored hot-rods
-with rocket-launchers and heavy machine-guns did battle on the American
-highways of the future. The lively Car Wars discussion on the Illuminati
-board featured many meticulous, painstaking discussions of the effects
-of grenades, land-mines, flamethrowers and napalm. It sounded like
-hacker anarchy files run amuck.
-
-Mr Jackson and his co-workers earned their daily bread by supplying people
-with make-believe adventures and weird ideas. The more far-out, the better.
-
-Simulation gaming is an unusual pastime, but gamers have not
-generally had to beg the permission of the Secret Service to exist.
-Wargames and role-playing adventures are an old and honored pastime,
-much favored by professional military strategists. Once little-known,
-these games are now played by hundreds of thousands of enthusiasts
-throughout North America, Europe and Japan. Gaming-books, once restricted
-to hobby outlets, now commonly appear in chain-stores like B. Dalton's
-and Waldenbooks, and sell vigorously.
-
-Steve Jackson Games, Inc., of Austin, Texas, was a games company
-of the middle rank. In 1989, SJG grossed about a million dollars.
-Jackson himself had a good reputation in his industry as a talented
-and innovative designer of rather unconventional games, but his company
-was something less than a titan of the field--certainly not like the
-multimillion-dollar TSR Inc., or Britain's gigantic "Games Workshop."
-SJG's Austin headquarters was a modest two-story brick office-suite,
-cluttered with phones, photocopiers, fax machines and computers.
-It bustled with semi-organized activity and was littered with
-glossy promotional brochures and dog-eared science-fiction novels.
-Attached to the offices was a large tin-roofed warehouse piled twenty feet
-high with cardboard boxes of games and books. Despite the weird imaginings
-that went on within it, the SJG headquarters was quite a quotidian,
-everyday sort of place. It looked like what it was: a publishers' digs.
-
-Both "Car Wars" and "Illuminati" were well-known, popular games.
-But the mainstay of the Jackson organization was their Generic Universal
-Role-Playing System, "G.U.R.P.S." The GURPS system was considered solid
-and well-designed, an asset for players. But perhaps the most popular
-feature of the GURPS system was that it allowed gaming-masters to design
-scenarios that closely resembled well-known books, movies, and other works
-of fantasy. Jackson had licensed and adapted works from many science fiction
-and fantasy authors. There was GURPS Conan, GURPS Riverworld,
-GURPS Horseclans, GURPS Witch World, names eminently familiar
-to science-fiction readers. And there was GURPS Special Ops,
-from the world of espionage fantasy and unconventional warfare.
-
-And then there was GURPS Cyberpunk.
-
-"Cyberpunk" was a term given to certain science fiction writers
-who had entered the genre in the 1980s. "Cyberpunk," as the label implies,
-had two general distinguishing features. First, its writers had a compelling
-interest in information technology, an interest closely akin
-to science fiction's earlier fascination with space travel.
-And second, these writers were "punks," with all the
-distinguishing features that that implies: Bohemian artiness,
-youth run wild, an air of deliberate rebellion, funny clothes and hair,
-odd politics, a fondness for abrasive rock and roll; in a word, trouble.
-
-The "cyberpunk" SF writers were a small group of mostly college-educated
-white middle-class litterateurs, scattered through the US and Canada.
-Only one, Rudy Rucker, a professor of computer science in Silicon Valley,
-could rank with even the humblest computer hacker. But, except for
-Professor Rucker, the "cyberpunk" authors were not programmers
-or hardware experts; they considered themselves artists
-(as, indeed, did Professor Rucker). However, these writers
-all owned computers, and took an intense and public interest
-in the social ramifications of the information industry.
-
-The cyberpunks had a strong following among the global generation
-that had grown up in a world of computers, multinational networks,
-and cable television. Their outlook was considered somewhat morbid,
-cynical, and dark, but then again, so was the outlook of their
-generational peers. As that generation matured and increased
-in strength and influence, so did the cyberpunks.
-As science-fiction writers went, they were doing
-fairly well for themselves. By the late 1980s,
-their work had attracted attention from gaming companies,
-including Steve Jackson Games, which was planning a cyberpunk
-simulation for the flourishing GURPS gaming-system.
-
-The time seemed ripe for such a product, which had already been proven
-in the marketplace. The first games- company out of the gate,
-with a product boldly called "Cyberpunk" in defiance of possible
-infringement-of-copyright suits, had been an upstart group called
-R. Talsorian. Talsorian's Cyberpunk was a fairly decent game,
-but the mechanics of the simulation system left a lot to be desired.
-Commercially, however, the game did very well.
-
-The next cyberpunk game had been the even more successful Shadowrun
-by FASA Corporation. The mechanics of this game were fine, but the
-scenario was rendered moronic by sappy fantasy elements like elves,
-trolls, wizards, and dragons--all highly ideologically-incorrect,
-according to the hard-edged, high-tech standards of cyberpunk science fiction.
-
-Other game designers were champing at the bit. Prominent among them
-was the Mentor, a gentleman who, like most of his friends in the
-Legion of Doom, was quite the cyberpunk devotee. Mentor reasoned
-that the time had come for a REAL cyberpunk gaming-book--one that the
-princes of computer-mischief in the Legion of Doom could play without
-laughing themselves sick. This book, GURPS Cyberpunk, would reek
-of culturally on-line authenticity.
-
-Mentor was particularly well-qualified for this task.
-Naturally, he knew far more about computer-intrusion
-and digital skullduggery than any previously published
-cyberpunk author. Not only that, but he was good at his work.
-A vivid imagination, combined with an instinctive feeling
-for the working of systems and, especially, the loopholes
-within them, are excellent qualities for a professional game designer.
-
-By March 1st, GURPS Cyberpunk was almost complete, ready to print and ship.
-Steve Jackson expected vigorous sales for this item, which, he hoped,
-would keep the company financially afloat for several months.
-GURPS Cyberpunk, like the other GURPS "modules," was not a "game"
-like a Monopoly set, but a BOOK: a bound paperback book the size
-of a glossy magazine, with a slick color cover, and pages full of text,
-illustrations, tables and footnotes. It was advertised as a game,
-and was used as an aid to game-playing, but it was a book,
-with an ISBN number, published in Texas, copyrighted,
-and sold in bookstores.
-
-And now, that book, stored on a computer, had gone out the door
-in the custody of the Secret Service.
-
-The day after the raid, Steve Jackson visited the local Secret Service
-headquarters with a lawyer in tow. There he confronted Tim Foley
-(still in Austin at that time) and demanded his book back. But there
-was trouble. GURPS Cyberpunk, alleged a Secret Service agent to astonished
-businessman Steve Jackson, was "a manual for computer crime."
-
-"It's science fiction," Jackson said.
-
-"No, this is real."
-
-This statement was repeated several times, by several agents.
-Jackson's ominously accurate game had passed from pure,
-obscure, small-scale fantasy into the impure, highly publicized,
-large-scale fantasy of the Hacker Crackdown.
-
-No mention was made of the real reason for the search.
-According to their search warrant, the raiders had expected
-to find the E911 Document stored on Jackson's bulletin board system.
-But that warrant was sealed; a procedure that most law enforcement agencies
-will use only when lives are demonstrably in danger. The raiders'
-true motives were not discovered until the Jackson search-warrant
-was unsealed by his lawyers, many months later. The Secret Service,
-and the Chicago Computer Fraud and Abuse Task Force,
-said absolutely nothing to Steve Jackson about any threat
-to the police 911 System. They said nothing about the Atlanta Three,
-nothing about Phrack or Knight Lightning, nothing about Terminus.
-
-Jackson was left to believe that his computers had been seized because
-he intended to publish a science fiction book that law enforcement
-considered too dangerous to see print.
-
-This misconception was repeated again and again, for months,
-to an ever-widening public audience. It was not the truth of the case;
-but as months passed, and this misconception was publicly printed again
-and again, it became one of the few publicly known "facts" about
-the mysterious Hacker Crackdown. The Secret Service had seized a computer
-to stop the publication of a cyberpunk science fiction book.
-
-The second section of this book, "The Digital Underground,"
-is almost finished now. We have become acquainted with all
-the major figures of this case who actually belong to the
-underground milieu of computer intrusion. We have some idea
-of their history, their motives, their general modus operandi.
-We now know, I hope, who they are, where they came from,
-and more or less what they want. In the next section of this book,
-"Law and Order," we will leave this milieu and directly enter the
-world of America's computer-crime police.
-
-At this point, however, I have another figure to introduce: myself.
-
-My name is Bruce Sterling. I live in Austin, Texas, where I am
-a science fiction writer by trade: specifically, a CYBERPUNK
-science fiction writer.
-
-Like my "cyberpunk" colleagues in the U.S. and Canada,
-I've never been entirely happy with this literary label--
-especially after it became a synonym for computer criminal.
-But I did once edit a book of stories by my colleagues,
-called Mirrorshades: the Cyberpunk Anthology, and I've
-long been a writer of literary-critical cyberpunk manifestos.
-I am not a "hacker" of any description, though I do have readers
-in the digital underground.
-
-When the Steve Jackson Games seizure occurred, I naturally took
-an intense interest. If "cyberpunk" books were being banned
-by federal police in my own home town, I reasonably wondered
-whether I myself might be next. Would my computer be seized
-by the Secret Service? At the time, I was in possession
-of an aging Apple IIe without so much as a hard disk.
-If I were to be raided as an author of computer-crime manuals,
-the loss of my feeble word-processor would likely provoke more
-snickers than sympathy.
-
-I'd known Steve Jackson for many years. We knew
-one another as colleagues, for we frequented
-the same local science-fiction conventions.
-I'd played Jackson games, and recognized his cleverness;
-but he certainly had never struck me as a potential mastermind
-of computer crime.
-
-I also knew a little about computer bulletin-board systems.
-In the mid-1980s I had taken an active role in an Austin board
-called "SMOF-BBS," one of the first boards dedicated to science fiction.
-I had a modem, and on occasion I'd logged on to Illuminati,
-which always looked entertainly wacky, but certainly harmless enough.
-
-At the time of the Jackson seizure, I had no experience
-whatsoever with underground boards. But I knew that no one
-on Illuminati talked about breaking into systems illegally,
-or about robbing phone companies. Illuminati didn't even
-offer pirated computer games. Steve Jackson, like many creative artists,
-was markedly touchy about theft of intellectual property.
-
-It seemed to me that Jackson was either seriously suspected
-of some crime--in which case, he would be charged soon,
-and would have his day in court--or else he was innocent,
-in which case the Secret Service would quickly return his equipment,
-and everyone would have a good laugh. I rather expected the good laugh.
-The situation was not without its comic side. The raid, known
-as the "Cyberpunk Bust" in the science fiction community,
-was winning a great deal of free national publicity both
-for Jackson himself and the "cyberpunk" science fiction
-writers generally.
-
-Besides, science fiction people are used to being misinterpreted.
-Science fiction is a colorful, disreputable, slipshod occupation,
-full of unlikely oddballs, which, of course, is why we like it.
-Weirdness can be an occupational hazard in our field. People who
-wear Halloween costumes are sometimes mistaken for monsters.
-
-Once upon a time--back in 1939, in New York City--
-science fiction and the U.S. Secret Service collided in
-a comic case of mistaken identity. This weird incident
-involved a literary group quite famous in science fiction,
-known as "the Futurians," whose membership included
-such future genre greats as Isaac Asimov, Frederik Pohl,
-and Damon Knight. The Futurians were every bit as
-offbeat and wacky as any of their spiritual descendants,
-including the cyberpunks, and were given to communal living,
-spontaneous group renditions of light opera, and midnight fencing
-exhibitions on the lawn. The Futurians didn't have bulletin
-board systems, but they did have the technological equivalent
-in 1939--mimeographs and a private printing press. These were
-in steady use, producing a stream of science-fiction fan magazines,
-literary manifestos, and weird articles, which were picked up
-in ink-sticky bundles by a succession of strange, gangly,
-spotty young men in fedoras and overcoats.
-
-The neighbors grew alarmed at the antics of the Futurians
-and reported them to the Secret Service as suspected counterfeiters.
-In the winter of 1939, a squad of USSS agents with drawn guns burst into
-"Futurian House," prepared to confiscate the forged currency and illicit
-printing presses. There they discovered a slumbering science fiction fan
-named George Hahn, a guest of the Futurian commune who had just arrived
-in New York. George Hahn managed to explain himself and his group,
-and the Secret Service agents left the Futurians in peace henceforth.
-(Alas, Hahn died in 1991, just before I had discovered this astonishing
-historical parallel, and just before I could interview him for this book.)
-
-But the Jackson case did not come to a swift and comic end.
-No quick answers came his way, or mine; no swift reassurances
-that all was right in the digital world, that matters were well
-in hand after all. Quite the opposite. In my alternate role
-as a sometime pop-science journalist, I interviewed Jackson
-and his staff for an article in a British magazine.
-The strange details of the raid left me more concerned than ever.
-Without its computers, the company had been financially
-and operationally crippled. Half the SJG workforce,
-a group of entirely innocent people, had been sorrowfully fired,
-deprived of their livelihoods by the seizure. It began to dawn on me
-that authors--American writers--might well have their computers seized,
-under sealed warrants, without any criminal charge; and that,
-as Steve Jackson had discovered, there was no immediate recourse for this.
-This was no joke; this wasn't science fiction; this was real.
-
-I determined to put science fiction aside until I had discovered
-what had happened and where this trouble had come from.
-It was time to enter the purportedly real world of electronic
-free expression and computer crime. Hence, this book.
-Hence, the world of the telcos; and the world of the digital underground;
-and next, the world of the police.
-
-
-
-PART THREE: LAW AND ORDER
-
-
-Of the various anti-hacker activities of 1990, "Operation Sundevil"
-had by far the highest public profile. The sweeping, nationwide
-computer seizures of May 8, 1990 were unprecedented in scope and highly,
-if rather selectively, publicized.
-
-Unlike the efforts of the Chicago Computer Fraud and Abuse Task Force,
-"Operation Sundevil" was not intended to combat "hacking" in the sense
-of computer intrusion or sophisticated raids on telco switching stations.
-Nor did it have anything to do with hacker misdeeds with AT&T's software,
-or with Southern Bell's proprietary documents.
-
-Instead, "Operation Sundevil" was a crackdown on those traditional scourges
-of the digital underground: credit-card theft and telephone code abuse.
-The ambitious activities out of Chicago, and the somewhat lesser-known
-but vigorous anti-hacker actions of the New York State Police in 1990,
-were never a part of "Operation Sundevil" per se, which was based in Arizona.
-
-Nevertheless, after the spectacular May 8 raids, the public, misled by
-police secrecy, hacker panic, and a puzzled national press-corps,
-conflated all aspects of the nationwide crackdown in 1990 under
-the blanket term "Operation Sundevil." "Sundevil" is still the best-known
-synonym for the crackdown of 1990. But the Arizona organizers of "Sundevil"
-did not really deserve this reputation--any more, for instance, than all
-hackers deserve a reputation as "hackers."
-
-There was some justice in this confused perception, though.
-For one thing, the confusion was abetted by the Washington office
-of the Secret Service, who responded to Freedom of Information Act
-requests on "Operation Sundevil" by referring investigators
-to the publicly known cases of Knight Lightning and the Atlanta Three.
-And "Sundevil" was certainly the largest aspect of the Crackdown,
-the most deliberate and the best-organized. As a crackdown on electronic
-fraud, "Sundevil" lacked the frantic pace of the war on the Legion of Doom;
-on the contrary, Sundevil's targets were picked out with cool deliberation
-over an elaborate investigation lasting two full years.
-
-And once again the targets were bulletin board systems.
-
-Boards can be powerful aids to organized fraud. Underground boards carry
-lively, extensive, detailed, and often quite flagrant "discussions" of
-lawbreaking techniques and lawbreaking activities. "Discussing" crime
-in the abstract, or "discussing" the particulars of criminal cases,
-is not illegal--but there are stern state and federal laws against
-coldbloodedly conspiring in groups in order to commit crimes.
-
-In the eyes of police, people who actively conspire to break the law
-are not regarded as "clubs," "debating salons," "users' groups," or
-"free speech advocates." Rather, such people tend to find themselves
-formally indicted by prosecutors as "gangs," "racketeers," "corrupt
-organizations" and "organized crime figures."
-
-What's more, the illicit data contained on outlaw boards goes well beyond
-mere acts of speech and/or possible criminal conspiracy. As we have seen,
-it was common practice in the digital underground to post purloined telephone
-codes on boards, for any phreak or hacker who cared to abuse them. Is posting
-digital booty of this sort supposed to be protected by the First Amendment?
-Hardly--though the issue, like most issues in cyberspace, is not entirely
-resolved. Some theorists argue that to merely RECITE a number publicly
-is not illegal--only its USE is illegal. But anti-hacker police point out
-that magazines and newspapers (more traditional forms of free expression)
-never publish stolen telephone codes (even though this might well
-raise their circulation).
-
-Stolen credit card numbers, being riskier and more valuable,
-were less often publicly posted on boards--but there is no question
-that some underground boards carried "carding" traffic,
-generally exchanged through private mail.
-
-Underground boards also carried handy programs for "scanning" telephone
-codes and raiding credit card companies, as well as the usual obnoxious
-galaxy of pirated software, cracked passwords, blue-box schematics,
-intrusion manuals, anarchy files, porn files, and so forth.
-
-But besides their nuisance potential for the spread of illicit knowledge,
-bulletin boards have another vitally interesting aspect for the
-professional investigator. Bulletin boards are cram-full of EVIDENCE.
-All that busy trading of electronic mail, all those hacker boasts,
-brags and struts, even the stolen codes and cards, can be neat,
-electronic, real-time recordings of criminal activity.
-As an investigator, when you seize a pirate board, you have
-scored a coup as effective as tapping phones or intercepting mail.
-However, you have not actually tapped a phone or intercepted a letter.
-The rules of evidence regarding phone-taps and mail interceptions are old,
-stern and well-understood by police, prosecutors and defense attorneys alike.
-The rules of evidence regarding boards are new, waffling, and understood
-by nobody at all.
-
-Sundevil was the largest crackdown on boards in world history.
-On May 7, 8, and 9, 1990, about forty-two computer systems were seized.
-Of those forty-two computers, about twenty-five actually were running boards.
-(The vagueness of this estimate is attributable to the vagueness of
-(a) what a "computer system" is, and (b) what it actually means to
-"run a board" with one--or with two computers, or with three.)
-
-About twenty-five boards vanished into police custody in May 1990.
-As we have seen, there are an estimated 30,000 boards in America today.
-If we assume that one board in a hundred is up to no good with codes
-and cards (which rather flatters the honesty of the board-using community),
-then that would leave 2,975 outlaw boards untouched by Sundevil.
-Sundevil seized about one tenth of one percent of all computer
-bulletin boards in America. Seen objectively, this is something less
-than a comprehensive assault. In 1990, Sundevil's organizers--
-the team at the Phoenix Secret Service office, and the Arizona
-Attorney General's office-- had a list of at least THREE HUNDRED
-boards that they considered fully deserving of search and seizure warrants.
-The twenty-five boards actually seized were merely among the most obvious
-and egregious of this much larger list of candidates. All these boards
-had been examined beforehand--either by informants, who had passed printouts
-to the Secret Service, or by Secret Service agents themselves, who not only
-come equipped with modems but know how to use them.
-
-There were a number of motives for Sundevil. First, it offered
-a chance to get ahead of the curve on wire-fraud crimes.
-Tracking back credit-card ripoffs to their perpetrators
-can be appallingly difficult. If these miscreants
-have any kind of electronic sophistication, they can snarl
-their tracks through the phone network into a mind-boggling,
-untraceable mess, while still managing to "reach out and rob someone."
-Boards, however, full of brags and boasts, codes and cards,
-offer evidence in the handy congealed form.
-
-Seizures themselves--the mere physical removal of machines--
-tends to take the pressure off. During Sundevil, a large number
-of code kids, warez d00dz, and credit card thieves would be deprived
-of those boards--their means of community and conspiracy--in one swift blow.
-As for the sysops themselves (commonly among the boldest offenders)
-they would be directly stripped of their computer equipment,
-and rendered digitally mute and blind.
-
-And this aspect of Sundevil was carried out with great success.
-Sundevil seems to have been a complete tactical surprise--
-unlike the fragmentary and continuing seizures of the war on the
-Legion of Doom, Sundevil was precisely timed and utterly overwhelming.
-At least forty "computers" were seized during May 7, 8 and 9, 1990,
-in Cincinnati, Detroit, Los Angeles, Miami, Newark, Phoenix, Tucson,
-Richmond, San Diego, San Jose, Pittsburgh and San Francisco.
-Some cities saw multiple raids, such as the five separate raids
-in the New York City environs. Plano, Texas (essentially a suburb of
-the Dallas/Fort Worth metroplex, and a hub of the telecommunications industry)
-saw four computer seizures. Chicago, ever in the forefront, saw its own
-local Sundevil raid, briskly carried out by Secret Service agents
-Timothy Foley and Barbara Golden.
-
-Many of these raids occurred, not in the cities proper,
-but in associated white-middle class suburbs--places like
-Mount Lebanon, Pennsylvania and Clark Lake, Michigan.
-There were a few raids on offices; most took place in people's homes,
-the classic hacker basements and bedrooms.
-
-The Sundevil raids were searches and seizures, not a group of mass arrests.
-There were only four arrests during Sundevil. "Tony the Trashman,"
-a longtime teenage bete noire of the Arizona Racketeering unit,
-was arrested in Tucson on May 9. "Dr. Ripco," sysop of an outlaw board
-with the misfortune to exist in Chicago itself, was also arrested--
-on illegal weapons charges. Local units also arrested a 19-year-old
-female phone phreak named "Electra" in Pennsylvania, and a male juvenile
-in California. Federal agents however were not seeking arrests, but computers.
-
-Hackers are generally not indicted (if at all) until the evidence
-in their seized computers is evaluated--a process that can take weeks,
-months--even years. When hackers are arrested on the spot, it's generally
-an arrest for other reasons. Drugs and/or illegal weapons show up in a good
-third of anti-hacker computer seizures (though not during Sundevil).
-
-That scofflaw teenage hackers (or their parents) should have marijuana
-in their homes is probably not a shocking revelation, but the surprisingly
-common presence of illegal firearms in hacker dens is a bit disquieting.
-A Personal Computer can be a great equalizer for the techno-cowboy--
-much like that more traditional American "Great Equalizer,"
-the Personal Sixgun. Maybe it's not all that surprising
-that some guy obsessed with power through illicit technology
-would also have a few illicit high-velocity-impact devices around.
-An element of the digital underground particularly dotes on those
-"anarchy philes," and this element tends to shade into the crackpot milieu
-of survivalists, gun-nuts, anarcho-leftists and the ultra-libertarian
-right-wing.
-
-This is not to say that hacker raids to date have uncovered any
-major crack-dens or illegal arsenals; but Secret Service agents
-do not regard "hackers" as "just kids." They regard hackers as
-unpredictable people, bright and slippery. It doesn't help matters
-that the hacker himself has been "hiding behind his keyboard"
-all this time. Commonly, police have no idea what he looks like.
-This makes him an unknown quantity, someone best treated with
-proper caution.
-
-To date, no hacker has come out shooting, though they do sometimes brag on
-boards that they will do just that. Threats of this sort are taken seriously.
-Secret Service hacker raids tend to be swift, comprehensive, well-manned
-(even over-manned); and agents generally burst through every door
-in the home at once, sometimes with drawn guns. Any potential resistance
-is swiftly quelled. Hacker raids are usually raids on people's homes.
-It can be a very dangerous business to raid an American home;
-people can panic when strangers invade their sanctum. Statistically speaking,
-the most dangerous thing a policeman can do is to enter someone's home.
-(The second most dangerous thing is to stop a car in traffic.)
-People have guns in their homes. More cops are hurt in homes
-than are ever hurt in biker bars or massage parlors.
-
-But in any case, no one was hurt during Sundevil,
-or indeed during any part of the Hacker Crackdown.
-
-Nor were there any allegations of any physical mistreatment of a suspect.
-Guns were pointed, interrogations were sharp and prolonged; but no one
-in 1990 claimed any act of brutality by any crackdown raider.
-
-In addition to the forty or so computers, Sundevil reaped floppy disks
-in particularly great abundance--an estimated 23,000 of them, which
-naturally included every manner of illegitimate data: pirated games,
-stolen codes, hot credit card numbers, the complete text and software
-of entire pirate bulletin-boards. These floppy disks, which remain
-in police custody today, offer a gigantic, almost embarrassingly
-rich source of possible criminal indictments. These 23,000 floppy disks
-also include a thus-far unknown quantity of legitimate computer games,
-legitimate software, purportedly "private" mail from boards,
-business records, and personal correspondence of all kinds.
-
-Standard computer-crime search warrants lay great emphasis on seizing
-written documents as well as computers--specifically including photocopies,
-computer printouts, telephone bills, address books, logs, notes,
-memoranda and correspondence. In practice, this has meant that diaries,
-gaming magazines, software documentation, nonfiction books on hacking
-and computer security, sometimes even science fiction novels, have all
-vanished out the door in police custody. A wide variety of electronic items
-have been known to vanish as well, including telephones, televisions, answering
-machines, Sony Walkmans, desktop printers, compact disks, and audiotapes.
-
-No fewer than 150 members of the Secret Service were sent into
-the field during Sundevil. They were commonly accompanied by
-squads of local and/or state police. Most of these officers--
-especially the locals--had never been on an anti-hacker raid before.
-(This was one good reason, in fact, why so many of them were invited along
-in the first place.) Also, the presence of a uniformed police officer
-assures the raidees that the people entering their homes are, in fact, police.
-Secret Service agents wear plain clothes. So do the telco security experts
-who commonly accompany the Secret Service on raids (and who make no particular
-effort to identify themselves as mere employees of telephone companies).
-
-A typical hacker raid goes something like this. First, police storm in
-rapidly, through every entrance, with overwhelming force,
-in the assumption that this tactic will keep casualties to a minimum.
-Second, possible suspects are immediately removed from the vicinity
-of any and all computer systems, so that they will have no chance
-to purge or destroy computer evidence. Suspects are herded into a room
-without computers, commonly the living room, and kept under guard--
-not ARMED guard, for the guns are swiftly holstered, but under guard
-nevertheless. They are presented with the search warrant and warned
-that anything they say may be held against them. Commonly they have
-a great deal to say, especially if they are unsuspecting parents.
-
-Somewhere in the house is the "hot spot"--a computer tied to a phone
-line (possibly several computers and several phones). Commonly it's
-a teenager's bedroom, but it can be anywhere in the house;
-there may be several such rooms. This "hot spot" is put in charge
-of a two-agent team, the "finder" and the "recorder." The "finder"
-is computer-trained, commonly the case agent who has actually obtained
-the search warrant from a judge. He or she understands what is being sought,
-and actually carries out the seizures: unplugs machines, opens drawers,
-desks, files, floppy-disk containers, etc. The "recorder" photographs
-all the equipment, just as it stands--especially the tangle of
-wired connections in the back, which can otherwise be a real nightmare
-to restore. The recorder will also commonly photograph every room
-in the house, lest some wily criminal claim that the police had robbed him
-during the search. Some recorders carry videocams or tape recorders;
-however, it's more common for the recorder to simply take written notes.
-Objects are described and numbered as the finder seizes them, generally
-on standard preprinted police inventory forms.
-
-Even Secret Service agents were not, and are not, expert computer users.
-They have not made, and do not make, judgements on the fly about potential
-threats posed by various forms of equipment. They may exercise discretion;
-they may leave Dad his computer, for instance, but they don't HAVE to.
-Standard computer-crime search warrants, which date back to the early 80s,
-use a sweeping language that targets computers, most anything attached
-to a computer, most anything used to operate a computer--most anything
-that remotely resembles a computer--plus most any and all written documents
-surrounding it. Computer-crime investigators have strongly urged agents
-to seize the works.
-
-In this sense, Operation Sundevil appears to have been a complete success.
-Boards went down all over America, and were shipped en masse to the computer
-investigation lab of the Secret Service, in Washington DC, along with the
-23,000 floppy disks and unknown quantities of printed material.
-
-But the seizure of twenty-five boards, and the multi-megabyte mountains
-of possibly useful evidence contained in these boards (and in their owners'
-other computers, also out the door), were far from the only motives for
-Operation Sundevil. An unprecedented action of great ambition and size,
-Sundevil's motives can only be described as political. It was a
-public-relations effort, meant to pass certain messages, meant to make
-certain situations clear: both in the mind of the general public,
-and in the minds of various constituencies of the electronic community.
-
- First --and this motivation was vital--a "message" would be sent from
-law enforcement to the digital underground. This very message was recited
-in so many words by Garry M. Jenkins, the Assistant Director of the
-US Secret Service, at the Sundevil press conference in Phoenix on
-May 9, 1990, immediately after the raids. In brief, hackers were
-mistaken in their foolish belief that they could hide behind the
-"relative anonymity of their computer terminals." On the contrary,
-they should fully understand that state and federal cops were
-actively patrolling the beat in cyberspace--that they were
-on the watch everywhere, even in those sleazy and secretive
-dens of cybernetic vice, the underground boards.
-
-This is not an unusual message for police to publicly convey to crooks.
-The message is a standard message; only the context is new.
-
-In this respect, the Sundevil raids were the digital equivalent
-of the standard vice-squad crackdown on massage parlors, porno bookstores,
-head-shops, or floating crap-games. There may be few or no arrests in a raid
-of this sort; no convictions, no trials, no interrogations. In cases of this
-sort, police may well walk out the door with many pounds of sleazy magazines,
-X-rated videotapes, sex toys, gambling equipment, baggies of marijuana. . . .
-
-Of course, if something truly horrendous is discovered by the raiders,
-there will be arrests and prosecutions. Far more likely, however,
-there will simply be a brief but sharp disruption of the closed
-and secretive world of the nogoodniks. There will be "street hassle."
-"Heat." "Deterrence." And, of course, the immediate loss of the seized goods.
-It is very unlikely that any of this seized material will ever be returned.
-Whether charged or not, whether convicted or not, the perpetrators will
-almost surely lack the nerve ever to ask for this stuff to be given back.
-
-Arrests and trials--putting people in jail--may involve all kinds of
-formal legalities; but dealing with the justice system is far from the only
-task of police. Police do not simply arrest people. They don't simply
-put people in jail. That is not how the police perceive their jobs.
-Police "protect and serve." Police "keep the peace," they "keep public order."
-Like other forms of public relations, keeping public order is not an
-exact science. Keeping public order is something of an art-form.
-
-If a group of tough-looking teenage hoodlums was loitering on a street-corner,
-no one would be surprised to see a street-cop arrive and sternly order
-them to "break it up." On the contrary, the surprise would come if one
-of these ne'er-do-wells stepped briskly into a phone-booth,
-called a civil rights lawyer, and instituted a civil suit
-in defense of his Constitutional rights of free speech
-and free assembly. But something much along this line
-was one of the many anomolous outcomes of the Hacker Crackdown.
-
-Sundevil also carried useful "messages" for other constituents of
-the electronic community. These messages may not have been read
-aloud from the Phoenix podium in front of the press corps,
-but there was little mistaking their meaning. There was a message
-of reassurance for the primary victims of coding and carding:
-the telcos, and the credit companies. Sundevil was greeted with joy
-by the security officers of the electronic business community.
-After years of high-tech harassment and spiralling revenue losses,
-their complaints of rampant outlawry were being taken seriously by
-law enforcement. No more head-scratching or dismissive shrugs;
-no more feeble excuses about "lack of computer-trained officers" or
-the low priority of "victimless" white-collar telecommunication crimes.
-
-Computer-crime experts have long believed that computer-related offenses
-are drastically under-reported. They regard this as a major open scandal
-of their field. Some victims are reluctant to come forth, because they
-believe that police and prosecutors are not computer-literate,
-and can and will do nothing. Others are embarrassed by
-their vulnerabilities, and will take strong measures
-to avoid any publicity; this is especially true of banks,
-who fear a loss of investor confidence should an embezzlement-case
-or wire-fraud surface. And some victims are so helplessly confused
-by their own high technology that they never even realize that
-a crime has occurred--even when they have been fleeced to the bone.
-
-The results of this situation can be dire.
-Criminals escape apprehension and punishment.
-The computer-crime units that do exist, can't get work.
-The true scope of computer-crime: its size, its real nature,
-the scope of its threats, and the legal remedies for it--
-all remain obscured.
-
-Another problem is very little publicized, but it is a cause
-of genuine concern. Where there is persistent crime,
-but no effective police protection, then vigilantism can result.
-Telcos, banks, credit companies, the major corporations who
-maintain extensive computer networks vulnerable to hacking
---these organizations are powerful, wealthy, and
-politically influential. They are disinclined to be
-pushed around by crooks (or by most anyone else,
-for that matter). They often maintain well-organized
-private security forces, commonly run by
-experienced veterans of military and police units,
-who have left public service for the greener pastures
-of the private sector. For police, the corporate
-security manager can be a powerful ally; but if this
-gentleman finds no allies in the police, and the
-pressure is on from his board-of-directors,
-he may quietly take certain matters into his own hands.
-
-Nor is there any lack of disposable hired-help in the
-corporate security business. Private security agencies--
-the `security business' generally--grew explosively in the 1980s.
-Today there are spooky gumshoed armies of "security consultants,"
-"rent-a- cops," "private eyes," "outside experts"--every manner
-of shady operator who retails in "results" and discretion.
-Or course, many of these gentlemen and ladies may be paragons
-of professional and moral rectitude. But as anyone
-who has read a hard-boiled detective novel knows,
-police tend to be less than fond of this sort
-of private-sector competition.
-
-Companies in search of computer-security have even been
-known to hire hackers. Police shudder at this prospect.
-
-Police treasure good relations with the business community.
-Rarely will you see a policeman so indiscreet as to allege
-publicly that some major employer in his state or city has succumbed
-to paranoia and gone off the rails. Nevertheless,
-police --and computer police in particular--are aware
-of this possibility. Computer-crime police can and do
-spend up to half of their business hours just doing
-public relations: seminars, "dog and pony shows,"
-sometimes with parents' groups or computer users,
-but generally with their core audience: the likely
-victims of hacking crimes. These, of course, are telcos,
-credit card companies and large computer-equipped corporations.
-The police strongly urge these people, as good citizens,
-to report offenses and press criminal charges;
-they pass the message that there is someone in authority who cares,
-understands, and, best of all, will take useful action
-should a computer-crime occur.
-
-But reassuring talk is cheap. Sundevil offered action.
-
-The final message of Sundevil was intended for internal consumption
-by law enforcement. Sundevil was offered as proof that the community
-of American computer-crime police had come of age. Sundevil was
-proof that enormous things like Sundevil itself could now be accomplished.
-Sundevil was proof that the Secret Service and its local law-enforcement
-allies could act like a well-oiled machine--(despite the hampering use
-of those scrambled phones). It was also proof that the Arizona Organized
-Crime and Racketeering Unit--the sparkplug of Sundevil--ranked with the best
-in the world in ambition, organization, and sheer conceptual daring.
-
-And, as a final fillip, Sundevil was a message from the Secret Service
-to their longtime rivals in the Federal Bureau of Investigation.
-By Congressional fiat, both USSS and FBI formally share jurisdiction
-over federal computer-crimebusting activities. Neither of these groups
-has ever been remotely happy with this muddled situation. It seems to
-suggest that Congress cannot make up its mind as to which of these groups
-is better qualified. And there is scarcely a G-man or a Special Agent
-anywhere without a very firm opinion on that topic.
-
-#
-
-For the neophyte, one of the most puzzling aspects of the crackdown
-on hackers is why the United States Secret Service has anything at all
-to do with this matter.
-
-The Secret Service is best known for its primary public role:
-its agents protect the President of the United States.
-They also guard the President's family, the Vice President and his family,
-former Presidents, and Presidential candidates. They sometimes guard
-foreign dignitaries who are visiting the United States, especially foreign
-heads of state, and have been known to accompany American officials
-on diplomatic missions overseas.
-
-Special Agents of the Secret Service don't wear uniforms, but the
-Secret Service also has two uniformed police agencies. There's the
-former White House Police (now known as the Secret Service Uniformed Division,
-since they currently guard foreign embassies in Washington, as well as the
-White House itself). And there's the uniformed Treasury Police Force.
-
-The Secret Service has been charged by Congress with a number
-of little-known duties. They guard the precious metals in Treasury vaults.
-They guard the most valuable historical documents of the United States:
-originals of the Constitution, the Declaration of Independence,
-Lincoln's Second Inaugural Address, an American-owned copy of
-the Magna Carta, and so forth. Once they were assigned to guard
-the Mona Lisa, on her American tour in the 1960s.
-
-The entire Secret Service is a division of the Treasury Department.
-Secret Service Special Agents (there are about 1,900 of them)
-are bodyguards for the President et al, but they all work for the Treasury.
-And the Treasury (through its divisions of the U.S. Mint and the
-Bureau of Engraving and Printing) prints the nation's money.
-
-As Treasury police, the Secret Service guards the nation's currency;
-it is the only federal law enforcement agency with direct jurisdiction
-over counterfeiting and forgery. It analyzes documents for authenticity,
-and its fight against fake cash is still quite lively (especially since
-the skilled counterfeiters of Medellin, Columbia have gotten into the act).
-Government checks, bonds, and other obligations, which exist in untold
-millions and are worth untold billions, are common targets for forgery,
-which the Secret Service also battles. It even handles forgery
-of postage stamps.
-
-But cash is fading in importance today as money has become electronic.
-As necessity beckoned, the Secret Service moved from fighting the
-counterfeiting of paper currency and the forging of checks,
-to the protection of funds transferred by wire.
-
-From wire-fraud, it was a simple skip-and-jump to what is formally
-known as "access device fraud." Congress granted the Secret Service
-the authority to investigate "access device fraud" under Title 18
-of the United States Code (U.S.C. Section 1029).
-
-The term "access device" seems intuitively simple. It's some kind
-of high-tech gizmo you use to get money with. It makes good sense
-to put this sort of thing in the charge of counterfeiting and
-wire-fraud experts.
-
-However, in Section 1029, the term "access device" is very
-generously defined. An access device is: "any card, plate,
-code, account number, or other means of account access
-that can be used, alone or in conjunction with another access device,
-to obtain money, goods, services, or any other thing of value,
-or that can be used to initiate a transfer of funds."
-
-"Access device" can therefore be construed to include credit cards
-themselves (a popular forgery item nowadays). It also includes credit card
-account NUMBERS, those standards of the digital underground. The same goes
-for telephone charge cards (an increasingly popular item with telcos,
-who are tired of being robbed of pocket change by phone-booth thieves).
-And also telephone access CODES, those OTHER standards of the digital
-underground. (Stolen telephone codes may not "obtain money," but they
-certainly do obtain valuable "services," which is specifically forbidden
-by Section 1029.)
-
-We can now see that Section 1029 already pits the United States Secret Service
-directly against the digital underground, without any mention at all of
-the word "computer."
-
-Standard phreaking devices, like "blue boxes," used to steal phone service
-from old-fashioned mechanical switches, are unquestionably "counterfeit
-access devices." Thanks to Sec.1029, it is not only illegal to USE
-counterfeit access devices, but it is even illegal to BUILD them.
-"Producing," "designing" "duplicating" or "assembling" blue boxes
-are all federal crimes today, and if you do this, the Secret Service
-has been charged by Congress to come after you.
-
-Automatic Teller Machines, which replicated all over America during the 1980s,
-are definitely "access devices," too, and an attempt to tamper with their
-punch-in codes and plastic bank cards falls directly under Sec. 1029.
-
-Section 1029 is remarkably elastic. Suppose you find a computer password
-in somebody's trash. That password might be a "code"--it's certainly a
-"means of account access." Now suppose you log on to a computer
-and copy some software for yourself. You've certainly obtained
-"service" (computer service) and a "thing of value" (the software).
-Suppose you tell a dozen friends about your swiped password,
-and let them use it, too. Now you're "trafficking in unauthorized
-access devices." And when the Prophet, a member of the Legion of Doom,
-passed a stolen telephone company document to Knight Lightning
-at Phrack magazine, they were both charged under Sec. 1029!
-
-There are two limitations on Section 1029. First, the offense must
-"affect interstate or foreign commerce" in order to become a matter
-of federal jurisdiction. The term "affecting commerce" is not well defined;
-but you may take it as a given that the Secret Service can take an interest
-if you've done most anything that happens to cross a state line.
-State and local police can be touchy about their jurisdictions,
-and can sometimes be mulish when the feds show up. But when it comes
-to computer-crime, the local police are pathetically grateful
-for federal help--in fact they complain that they can't get enough of it.
-If you're stealing long-distance service, you're almost certainly crossing
-state lines, and you're definitely "affecting the interstate commerce"
-of the telcos. And if you're abusing credit cards by ordering stuff
-out of glossy catalogs from, say, Vermont, you're in for it.
-
-The second limitation is money. As a rule, the feds don't pursue
-penny-ante offenders. Federal judges will dismiss cases that appear
-to waste their time. Federal crimes must be serious; Section 1029
-specifies a minimum loss of a thousand dollars.
-
-We now come to the very next section of Title 18, which is Section 1030,
-"Fraud and related activity in connection with computers." This statute
-gives the Secret Service direct jurisdiction over acts of computer intrusion.
-On the face of it, the Secret Service would now seem to command the field.
-Section 1030, however, is nowhere near so ductile as Section 1029.
-
-The first annoyance is Section 1030(d), which reads:
-
-"(d) The United States Secret Service shall,
-IN ADDITION TO ANY OTHER AGENCY HAVING SUCH AUTHORITY,
-have the authority to investigate offenses under this section.
-Such authority of the United States Secret Service shall be
-exercised in accordance with an agreement which shall be entered
-into by the Secretary of the Treasury AND THE ATTORNEY GENERAL."
-(Author's italics.) [Represented by capitals.]
-
-The Secretary of the Treasury is the titular head of the Secret Service,
-while the Attorney General is in charge of the FBI. In Section (d),
-Congress shrugged off responsibility for the computer-crime turf-battle
-between the Service and the Bureau, and made them fight it out all
-by themselves. The result was a rather dire one for the Secret Service,
-for the FBI ended up with exclusive jurisdiction over computer break-ins
-having to do with national security, foreign espionage, federally insured
-banks, and U.S. military bases, while retaining joint jurisdiction over
-all the other computer intrusions. Essentially, when it comes to Section 1030,
-the FBI not only gets the real glamor stuff for itself, but can peer over the
-shoulder of the Secret Service and barge in to meddle whenever it suits them.
-
-The second problem has to do with the dicey term
-"Federal interest computer." Section 1030(a)(2)
-makes it illegal to "access a computer without authorization"
-if that computer belongs to a financial institution or an issuer
-of credit cards (fraud cases, in other words). Congress was quite
-willing to give the Secret Service jurisdiction over
-money-transferring computers, but Congress balked at
-letting them investigate any and all computer intrusions.
-Instead, the USSS had to settle for the money machines
-and the "Federal interest computers." A "Federal interest computer"
-is a computer which the government itself owns, or is using.
-Large networks of interstate computers, linked over state lines,
-are also considered to be of "Federal interest." (This notion of
-"Federal interest" is legally rather foggy and has never been
-clearly defined in the courts. The Secret Service has never yet
-had its hand slapped for investigating computer break-ins that were NOT
-of "Federal interest," but conceivably someday this might happen.)
-
-So the Secret Service's authority over "unauthorized access"
-to computers covers a lot of territory, but by no means the
-whole ball of cyberspatial wax. If you are, for instance,
-a LOCAL computer retailer, or the owner of a LOCAL bulletin
-board system, then a malicious LOCAL intruder can break in,
-crash your system, trash your files and scatter viruses,
-and the U.S. Secret Service cannot do a single thing about it.
-
-At least, it can't do anything DIRECTLY. But the Secret Service
-will do plenty to help the local people who can.
-
-The FBI may have dealt itself an ace off the bottom of the deck
-when it comes to Section 1030; but that's not the whole story;
-that's not the street. What's Congress thinks is one thing,
-and Congress has been known to change its mind. The REAL
-turf-struggle is out there in the streets where it's happening.
-If you're a local street-cop with a computer problem,
-the Secret Service wants you to know where you can find
-the real expertise. While the Bureau crowd are off having
-their favorite shoes polished--(wing-tips)--and making derisive
-fun of the Service's favorite shoes--("pansy-ass tassels")--
-the tassel-toting Secret Service has a crew of ready-and-able
-hacker-trackers installed in the capital of every state in the Union.
-Need advice? They'll give you advice, or at least point you in
-the right direction. Need training? They can see to that, too.
-
-If you're a local cop and you call in the FBI, the FBI
-(as is widely and slanderously rumored) will order you around
-like a coolie, take all the credit for your busts,
-and mop up every possible scrap of reflected glory.
-The Secret Service, on the other hand, doesn't brag a lot.
-They're the quiet types. VERY quiet. Very cool. Efficient.
-High-tech. Mirrorshades, icy stares, radio ear-plugs,
-an Uzi machine-pistol tucked somewhere in that well-cut jacket.
-American samurai, sworn to give their lives to protect our President.
-"The granite agents." Trained in martial arts, absolutely fearless.
-Every single one of 'em has a top-secret security clearance.
-Something goes a little wrong, you're not gonna hear any whining
-and moaning and political buck-passing out of these guys.
-
-The facade of the granite agent is not, of course, the reality.
-Secret Service agents are human beings. And the real glory
-in Service work is not in battling computer crime--not yet,
-anyway--but in protecting the President. The real glamour
-of Secret Service work is in the White House Detail.
-If you're at the President's side, then the kids and the wife
-see you on television; you rub shoulders with the most powerful
-people in the world. That's the real heart of Service work,
-the number one priority. More than one computer investigation
-has stopped dead in the water when Service agents vanished at
-the President's need.
-
-There's romance in the work of the Service. The intimate access
-to circles of great power; the esprit-de-corps of a highly trained
-and disciplined elite; the high responsibility of defending the
-Chief Executive; the fulfillment of a patriotic duty. And as police
-work goes, the pay's not bad. But there's squalor in Service work, too.
-You may get spat upon by protesters howling abuse--and if they get violent,
-if they get too close, sometimes you have to knock one of them down--
-discreetly.
-
-The real squalor in Service work is drudgery such as "the quarterlies,"
-traipsing out four times a year, year in, year out, to interview the various
-pathetic wretches, many of them in prisons and asylums, who have seen fit
-to threaten the President's life. And then there's the grinding stress
-of searching all those faces in the endless bustling crowds, looking for
-hatred, looking for psychosis, looking for the tight, nervous face
-of an Arthur Bremer, a Squeaky Fromme, a Lee Harvey Oswald.
-It's watching all those grasping, waving hands for sudden movements,
-while your ears strain at your radio headphone for the long-rehearsed
-cry of "Gun!"
-
-It's poring, in grinding detail, over the biographies of every rotten
-loser who ever shot at a President. It's the unsung work of the
-Protective Research Section, who study scrawled, anonymous death threats
-with all the meticulous tools of anti-forgery techniques.
-
-And it's maintaining the hefty computerized files on anyone
-who ever threatened the President's life. Civil libertarians
-have become increasingly concerned at the Government's use
-of computer files to track American citizens--but the
-Secret Service file of potential Presidential assassins,
-which has upward of twenty thousand names, rarely causes
-a peep of protest. If you EVER state that you intend to
-kill the President, the Secret Service will want to know
-and record who you are, where you are, what you are,
-and what you're up to. If you're a serious threat--
-if you're officially considered "of protective interest"--
-then the Secret Service may well keep tabs on you
-for the rest of your natural life.
-
-Protecting the President has first call on all the Service's resources.
-But there's a lot more to the Service's traditions and history than
-standing guard outside the Oval Office.
-
-The Secret Service is the nation's oldest general federal
-law-enforcement agency. Compared to the Secret Service,
-the FBI are new-hires and the CIA are temps. The Secret Service
-was founded 'way back in 1865, at the suggestion of Hugh McCulloch,
-Abraham Lincoln's Secretary of the Treasury. McCulloch wanted
-a specialized Treasury police to combat counterfeiting.
-Abraham Lincoln agreed that this seemed a good idea, and,
-with a terrible irony, Abraham Lincoln was shot that
-very night by John Wilkes Booth.
-
-The Secret Service originally had nothing to do with protecting Presidents.
-They didn't take this on as a regular assignment until after the Garfield
-assassination in 1881. And they didn't get any Congressional money for it
-until President McKinley was shot in 1901. The Service was originally
-designed for one purpose: destroying counterfeiters.
-
-#
-
-There are interesting parallels between the Service's
-nineteenth-century entry into counterfeiting,
-and America's twentieth-century entry into computer-crime.
-
-In 1865, America's paper currency was a terrible muddle.
-Security was drastically bad. Currency was printed on the spot
-by local banks in literally hundreds of different designs.
-No one really knew what the heck a dollar bill was supposed to look like.
-Bogus bills passed easily. If some joker told you that a one-dollar bill
-from the Railroad Bank of Lowell, Massachusetts had a woman leaning on
-a shield, with a locomotive, a cornucopia, a compass, various agricultural
-implements, a railroad bridge, and some factories, then you pretty much had
-to take his word for it. (And in fact he was telling the truth!)
-
-SIXTEEN HUNDRED local American banks designed and printed their own
-paper currency, and there were no general standards for security.
-Like a badly guarded node in a computer network, badly designed bills
-were easy to fake, and posed a security hazard for the entire monetary system.
-
-No one knew the exact extent of the threat to the currency.
-There were panicked estimates that as much as a third of
-the entire national currency was faked. Counterfeiters--
-known as "boodlers" in the underground slang of the time--
-were mostly technically skilled printers who had gone to the bad.
-Many had once worked printing legitimate currency.
-Boodlers operated in rings and gangs. Technical experts
-engraved the bogus plates--commonly in basements in New York City.
-Smooth confidence men passed large wads of high-quality,
-high-denomination fakes, including the really sophisticated stuff--
-government bonds, stock certificates, and railway shares.
-Cheaper, botched fakes were sold or sharewared to low-level
-gangs of boodler wannabes. (The really cheesy lowlife boodlers
-merely upgraded real bills by altering face values,
-changing ones to fives, tens to hundreds, and so on.)
-
-The techniques of boodling were little-known and regarded
-with a certain awe by the mid- nineteenth-century public.
-The ability to manipulate the system for rip-off seemed
-diabolically clever. As the skill and daring of the
-boodlers increased, the situation became intolerable.
-The federal government stepped in, and began offering
-its own federal currency, which was printed in fancy green ink,
-but only on the back--the original "greenbacks." And at first,
-the improved security of the well-designed, well-printed
-federal greenbacks seemed to solve the problem; but then
-the counterfeiters caught on. Within a few years things were
-worse than ever: a CENTRALIZED system where ALL security was bad!
-
-The local police were helpless. The Government tried offering
-blood money to potential informants, but this met with little success.
-Banks, plagued by boodling, gave up hope of police help and hired
-private security men instead. Merchants and bankers queued up
-by the thousands to buy privately-printed manuals on currency security,
-slim little books like Laban Heath's INFALLIBLE GOVERNMENT
-COUNTERFEIT DETECTOR. The back of the book offered Laban Heath's
-patent microscope for five bucks.
-
-Then the Secret Service entered the picture. The first agents
-were a rough and ready crew. Their chief was one William P. Wood,
-a former guerilla in the Mexican War who'd won a reputation busting
-contractor fraudsters for the War Department during the Civil War.
-Wood, who was also Keeper of the Capital Prison, had a sideline
-as a counterfeiting expert, bagging boodlers for the federal bounty money.
-
-Wood was named Chief of the new Secret Service in July 1865.
-There were only ten Secret Service agents in all: Wood himself,
-a handful who'd worked for him in the War Department, and a few
-former private investigators--counterfeiting experts--whom Wood
-had won over to public service. (The Secret Service of 1865 was
-much the size of the Chicago Computer Fraud Task Force or the
-Arizona Racketeering Unit of 1990.) These ten "Operatives"
-had an additional twenty or so "Assistant Operatives" and "Informants."
-Besides salary and per diem, each Secret Service employee received
-a whopping twenty-five dollars for each boodler he captured.
-
-Wood himself publicly estimated that at least HALF of America's currency
-was counterfeit, a perhaps pardonable perception. Within a year the
-Secret Service had arrested over 200 counterfeiters. They busted about
-two hundred boodlers a year for four years straight.
-
-Wood attributed his success to travelling fast and light, hitting the
-bad-guys hard, and avoiding bureaucratic baggage. "Because my raids
-were made without military escort and I did not ask the assistance
-of state officers, I surprised the professional counterfeiter."
-
-Wood's social message to the once-impudent boodlers bore an eerie ring
-of Sundevil: "It was also my purpose to convince such characters that
-it would no longer be healthy for them to ply their vocation without
-being handled roughly, a fact they soon discovered."
-
-William P. Wood, the Secret Service's guerilla pioneer,
-did not end well. He succumbed to the lure of aiming for
-the really big score. The notorious Brockway Gang of New York City,
-headed by William E. Brockway, the "King of the Counterfeiters,"
-had forged a number of government bonds. They'd passed these
-brilliant fakes on the prestigious Wall Street investment
-firm of Jay Cooke and Company. The Cooke firm were frantic
-and offered a huge reward for the forgers' plates.
-
-Laboring diligently, Wood confiscated the plates
-(though not Mr. Brockway) and claimed the reward.
-But the Cooke company treacherously reneged.
-Wood got involved in a down-and-dirty lawsuit
-with the Cooke capitalists. Wood's boss,
-Secretary of the Treasury McCulloch, felt that
-Wood's demands for money and glory were unseemly,
-and even when the reward money finally came through,
-McCulloch refused to pay Wood anything.
-Wood found himself mired in a seemingly endless
-round of federal suits and Congressional lobbying.
-
-Wood never got his money. And he lost his job to boot.
-He resigned in 1869.
-
-Wood's agents suffered, too. On May 12, 1869, the second Chief
-of the Secret Service took over, and almost immediately fired
-most of Wood's pioneer Secret Service agents: Operatives,
-Assistants and Informants alike. The practice of receiving $25
-per crook was abolished. And the Secret Service began the long,
-uncertain process of thorough professionalization.
-
-Wood ended badly. He must have felt stabbed in the back.
-In fact his entire organization was mangled.
-
-On the other hand, William P. Wood WAS the first head of the Secret Service.
-William Wood was the pioneer. People still honor his name. Who remembers
-the name of the SECOND head of the Secret Service?
-
-As for William Brockway (also known as "Colonel Spencer"),
-he was finally arrested by the Secret Service in 1880.
-He did five years in prison, got out, and was still boodling
-at the age of seventy-four.
-
-#
-
-Anyone with an interest in Operation Sundevil--
-or in American computer-crime generally--
-could scarcely miss the presence of Gail Thackeray,
-Assistant Attorney General of the State of Arizona.
-Computer-crime training manuals often cited
-Thackeray's group and her work; she was the
-highest-ranking state official to specialize
-in computer-related offenses. Her name had been
-on the Sundevil press release (though modestly ranked
-well after the local federal prosecuting attorney and
-the head of the Phoenix Secret Service office).
-
-As public commentary, and controversy, began to mount
-about the Hacker Crackdown, this Arizonan state official
-began to take a higher and higher public profile.
-Though uttering almost nothing specific about
-the Sundevil operation itself, she coined some
-of the most striking soundbites of the growing propaganda war:
-"Agents are operating in good faith, and I don't think
-you can say that for the hacker community," was one.
-Another was the memorable "I am not a mad dog prosecutor"
-(Houston Chronicle, Sept 2, 1990.) In the meantime,
-the Secret Service maintained its usual extreme discretion;
-the Chicago Unit, smarting from the backlash
-of the Steve Jackson scandal, had gone completely to earth.
-
-As I collated my growing pile of newspaper clippings,
-Gail Thackeray ranked as a comparative fount of public
-knowledge on police operations.
-
-I decided that I had to get to know Gail Thackeray.
-I wrote to her at the Arizona Attorney General's Office.
-Not only did she kindly reply to me, but, to my astonishment,
-she knew very well what "cyberpunk" science fiction was.
-
-Shortly after this, Gail Thackeray lost her job.
-And I temporarily misplaced my own career as
-a science-fiction writer, to become a full-time
-computer-crime journalist. In early March, 1991,
-I flew to Phoenix, Arizona, to interview Gail Thackeray
-for my book on the hacker crackdown.
-
-#
-
-"Credit cards didn't used to cost anything to get,"
-says Gail Thackeray. "Now they cost forty bucks--
-and that's all just to cover the costs from RIP-OFF ARTISTS."
-
-Electronic nuisance criminals are parasites.
-One by one they're not much harm, no big deal.
-But they never come just one by one. They come in swarms,
-heaps, legions, sometimes whole subcultures. And they bite.
-Every time we buy a credit card today, we lose a little financial
-vitality to a particular species of bloodsucker.
-
-What, in her expert opinion, are the worst forms of electronic crime,
-I ask, consulting my notes. Is it--credit card fraud? Breaking into
-ATM bank machines? Phone-phreaking? Computer intrusions?
-Software viruses? Access-code theft? Records tampering?
-Software piracy? Pornographic bulletin boards?
-Satellite TV piracy? Theft of cable service?
-It's a long list. By the time I reach the end
-of it I feel rather depressed.
-
-"Oh no," says Gail Thackeray, leaning forward over the table,
-her whole body gone stiff with energetic indignation,
-"the biggest damage is telephone fraud. Fake sweepstakes,
-fake charities. Boiler-room con operations. You could pay off
-the national debt with what these guys steal. . . .
-They target old people, they get hold of credit ratings
-and demographics, they rip off the old and the weak."
-The words come tumbling out of her.
-
-It's low-tech stuff, your everyday boiler-room fraud.
-Grifters, conning people out of money over the phone,
-have been around for decades. This is where the word "phony" came from!
-
-It's just that it's so much EASIER now, horribly facilitated by advances
-in technology and the byzantine structure of the modern phone system.
-The same professional fraudsters do it over and over, Thackeray tells me,
-they hide behind dense onion-shells of fake companies. . . fake holding
-corporations nine or ten layers deep, registered all over the map.
-They get a phone installed under a false name in an empty safe-house.
-And then they call-forward everything out of that phone to yet
-another phone, a phone that may even be in another STATE.
-And they don't even pay the charges on their phones;
-after a month or so, they just split; set up somewhere else
-in another Podunkville with the same seedy crew of veteran phone-crooks.
-They buy or steal commercial credit card reports, slap them on the PC,
-have a program pick out people over sixty-five who pay a lot to charities.
-A whole subculture living off this, merciless folks on the con.
-
-"The `light-bulbs for the blind' people," Thackeray muses,
-with a special loathing. "There's just no end to them."
-
-We're sitting in a downtown diner in Phoenix, Arizona.
-It's a tough town, Phoenix. A state capital seeing some hard times.
-Even to a Texan like myself, Arizona state politics seem rather baroque.
-There was, and remains, endless trouble over the Martin Luther King holiday,
-the sort of stiff-necked, foot-shooting incident for which Arizona politics
-seem famous. There was Evan Mecham, the eccentric Republican millionaire
-governor who was impeached, after reducing state government to a
-ludicrous shambles. Then there was the national Keating scandal,
-involving Arizona savings and loans, in which both of Arizona's
-U.S. senators, DeConcini and McCain, played sadly prominent roles.
-
-And the very latest is the bizarre AzScam case,
-in which state legislators were videotaped,
-eagerly taking cash from an informant of the Phoenix city
-police department, who was posing as a Vegas mobster.
-
-"Oh," says Thackeray cheerfully. "These people are amateurs here,
-they thought they were finally getting to play with the big boys.
-They don't have the least idea how to take a bribe!
-It's not institutional corruption. It's not like back in Philly."
-
-Gail Thackeray was a former prosecutor in Philadelphia.
-Now she's a former assistant attorney general of the State of Arizona.
-Since moving to Arizona in 1986, she had worked under the aegis
-of Steve Twist, her boss in the Attorney General's office.
-Steve Twist wrote Arizona's pioneering computer crime laws
-and naturally took an interest in seeing them enforced.
-It was a snug niche, and Thackeray's Organized Crime and
-Racketeering Unit won a national reputation for ambition
-and technical knowledgeability. . . . Until the latest
-election in Arizona. Thackeray's boss ran for the top
-job, and lost. The victor, the new Attorney General,
-apparently went to some pains to eliminate the bureaucratic
-traces of his rival, including his pet group--Thackeray's group.
-Twelve people got their walking papers.
-
-Now Thackeray's painstakingly assembled computer lab
-sits gathering dust somewhere in the glass-and-concrete
-Attorney General's HQ on 1275 Washington Street.
-Her computer-crime books, her painstakingly garnered
-back issues of phreak and hacker zines, all bought
-at her own expense--are piled in boxes somewhere.
-The State of Arizona is simply not particularly
-interested in electronic racketeering at the moment.
-
-At the moment of our interview, Gail Thackeray,
-officially unemployed, is working out of the county
-sheriff's office, living on her savings, and prosecuting
-several cases--working 60-hour weeks, just as always--
-for no pay at all. "I'm trying to train people,"
-she mutters.
-
-Half her life seems to be spent training people--merely pointing out,
-to the naive and incredulous (such as myself) that this stuff
-is ACTUALLY GOING ON OUT THERE. It's a small world, computer crime.
-A young world. Gail Thackeray, a trim blonde Baby-Boomer who favors
-Grand Canyon white-water rafting to kill some slow time,
-is one of the world's most senior, most veteran "hacker-trackers."
-Her mentor was Donn Parker, the California think-tank theorist
-who got it all started `way back in the mid-70s, the "grandfather
-of the field," "the great bald eagle of computer crime."
-
-And what she has learned, Gail Thackeray teaches. Endlessly.
-Tirelessly. To anybody. To Secret Service agents and state police,
-at the Glynco, Georgia federal training center. To local police,
-on "roadshows" with her slide projector and notebook.
-To corporate security personnel. To journalists. To parents.
-
-Even CROOKS look to Gail Thackeray for advice.
-Phone-phreaks call her at the office. They know very
-well who she is. They pump her for information
-on what the cops are up to, how much they know.
-Sometimes whole CROWDS of phone phreaks,
-hanging out on illegal conference calls, will call Gail
-Thackeray up. They taunt her. And, as always,
-they boast. Phone-phreaks, real stone phone-phreaks,
-simply CANNOT SHUT UP. They natter on for hours.
-
-Left to themselves, they mostly talk about the intricacies
-of ripping-off phones; it's about as interesting as listening
-to hot-rodders talk about suspension and distributor-caps.
-They also gossip cruelly about each other. And when talking
-to Gail Thackeray, they incriminate themselves. "I have tapes,"
-Thackeray says coolly.
-
-Phone phreaks just talk like crazy. "Dial-Tone" out in Alabama
-has been known to spend half-an-hour simply reading stolen
-phone-codes aloud into voice-mail answering machines.
-Hundreds, thousands of numbers, recited in a monotone,
-without a break--an eerie phenomenon. When arrested,
-it's a rare phone phreak who doesn't inform at endless length
-on everybody he knows.
-
-Hackers are no better. What other group of criminals,
-she asks rhetorically, publishes newsletters and holds conventions?
-She seems deeply nettled by the sheer brazenness of this behavior,
-though to an outsider, this activity might make one wonder
-whether hackers should be considered "criminals" at all.
-Skateboarders have magazines, and they trespass a lot.
-Hot rod people have magazines and they break speed limits
-and sometimes kill people. . . .
-
-I ask her whether it would be any loss to society if phone phreaking
-and computer hacking, as hobbies, simply dried up and blew away,
-so that nobody ever did it again.
-
-She seems surprised. "No," she says swiftly. "Maybe a little. . .
-in the old days. . .the MIT stuff. . . . But there's a lot of wonderful,
-legal stuff you can do with computers now, you don't have to break into
-somebody else's just to learn. You don't have that excuse.
-You can learn all you like."
-
-Did you ever hack into a system? I ask.
-
-The trainees do it at Glynco. Just to demonstrate system vulnerabilities.
-She's cool to the notion. Genuinely indifferent.
-
-"What kind of computer do you have?"
-
-"A Compaq 286LE," she mutters.
-
-"What kind do you WISH you had?"
-
-At this question, the unmistakable light of true hackerdom flares in
-Gail Thackeray's eyes. She becomes tense, animated, the words pour out:
-"An Amiga 2000 with an IBM card and Mac emulation! The most common hacker
-machines are Amigas and Commodores. And Apples." If she had the Amiga,
-she enthuses, she could run a whole galaxy of seized computer-evidence disks
-on one convenient multifunctional machine. A cheap one, too. Not like the
-old Attorney General lab, where they had an ancient CP/M machine,
-assorted Amiga flavors and Apple flavors, a couple IBMS, all the
-utility software. . .but no Commodores. The workstations down
-at the Attorney General's are Wang dedicated word-processors.
-Lame machines tied in to an office net--though at least they get
-on- line to the Lexis and Westlaw legal data services.
-
-I don't say anything. I recognize the syndrome, though.
-This computer-fever has been running through segments of
-our society for years now. It's a strange kind of lust:
-K-hunger, Meg-hunger; but it's a shared disease;
-it can kill parties dead, as conversation spirals into
-the deepest and most deviant recesses of software releases
-and expensive peripherals. . . . The mark of the hacker beast.
-I have it too. The whole "electronic community," whatever the hell
-that is, has it. Gail Thackeray has it. Gail Thackeray is a hacker cop.
-My immediate reaction is a strong rush of indignant pity:
-WHY DOESN'T SOMEBODY BUY THIS WOMAN HER AMIGA?!
-It's not like she's asking for a Cray X-MP
-supercomputer mainframe; an Amiga's a sweet little
-cookie-box thing. We're losing zillions in organized fraud;
-prosecuting and defending a single hacker case in court can cost
-a hundred grand easy. How come nobody can come up with four lousy grand
-so this woman can do her job? For a hundred grand we could buy every
-computer cop in America an Amiga. There aren't that many of 'em.
-
-Computers. The lust, the hunger, for computers.
-The loyalty they inspire, the intense sense of possessiveness.
-The culture they have bred. I myself am sitting in downtown Phoenix,
-Arizona because it suddenly occurred to me that the police might--
-just MIGHT--come and take away my computer. The prospect of this,
-the mere IMPLIED THREAT, was unbearable. It literally changed my life.
-It was changing the lives of many others. Eventually it would change
-everybody's life.
-
-Gail Thackeray was one of the top computer-crime people in America.
-And I was just some novelist, and yet I had a better computer than hers.
-PRACTICALLY EVERYBODY I KNEW had a better computer than Gail Thackeray
-and her feeble laptop 286. It was like sending the sheriff in to clean
-up Dodge City and arming her with a slingshot cut from an old rubber tire.
-
-But then again, you don't need a howitzer to enforce the law.
-You can do a lot just with a badge. With a badge alone,
-you can basically wreak havoc, take a terrible vengeance on wrongdoers.
-Ninety percent of "computer crime investigation" is just "crime investigation:"
-names, places, dossiers, modus operandi, search warrants, victims,
-complainants, informants. . . .
-
-What will computer crime look like in ten years? Will it get better?
-Did "Sundevil" send 'em reeling back in confusion?
-
-It'll be like it is now, only worse, she tells me with perfect conviction.
-Still there in the background, ticking along, changing with the times:
-the criminal underworld. It'll be like drugs are. Like our problems
-with alcohol. All the cops and laws in the world never solved our problems
-with alcohol. If there's something people want, a certain percentage
-of them are just going to take it. Fifteen percent of the populace
-will never steal. Fifteen percent will steal most anything not nailed down.
-The battle is for the hearts and minds of the remaining seventy percent.
-
-And criminals catch on fast. If there's not "too steep a learning curve"--
-if it doesn't require a baffling amount of expertise and practice--
-then criminals are often some of the first through the gate of a
-new technology. Especially if it helps them to hide.
-They have tons of cash, criminals. The new communications tech--
-like pagers, cellular phones, faxes, Federal Express--were pioneered
-by rich corporate people, and by criminals. In the early years
-of pagers and beepers, dope dealers were so enthralled this technology
-that owing a beeper was practically prima facie evidence of cocaine dealing.
-CB radio exploded when the speed limit hit 55 and breaking the highway law
-became a national pastime. Dope dealers send cash by Federal Express,
-despite, or perhaps BECAUSE OF, the warnings in FedEx offices that tell you
-never to try this. Fed Ex uses X-rays and dogs on their mail,
-to stop drug shipments. That doesn't work very well.
-
-Drug dealers went wild over cellular phones.
-There are simple methods of faking ID on cellular phones,
-making the location of the call mobile, free of charge,
-and effectively untraceable. Now victimized cellular
-companies routinely bring in vast toll-lists of calls
-to Colombia and Pakistan.
-
-Judge Greene's fragmentation of the phone company
-is driving law enforcement nuts. Four thousand
-telecommunications companies. Fraud skyrocketing.
-Every temptation in the world available with a phone
-and a credit card number. Criminals untraceable.
-A galaxy of "new neat rotten things to do."
-
-If there were one thing Thackeray would like to have,
-it would be an effective legal end-run through this new
-fragmentation minefield.
-
-It would be a new form of electronic search warrant,
-an "electronic letter of marque" to be issued by a judge.
-It would create a new category of "electronic emergency."
-Like a wiretap, its use would be rare, but it would cut
-across state lines and force swift cooperation from all concerned.
-Cellular, phone, laser, computer network, PBXes, AT&T, Baby Bells,
-long-distance entrepreneurs, packet radio. Some document,
-some mighty court-order, that could slice through four thousand
-separate forms of corporate red-tape, and get her at once to
-the source of calls, the source of email threats and viruses,
-the sources of bomb threats, kidnapping threats. "From now on,"
-she says, "the Lindbergh baby will always die."
-
-Something that would make the Net sit still, if only for a moment.
-Something that would get her up to speed. Seven league boots.
-That's what she really needs. "Those guys move in nanoseconds
-and I'm on the Pony Express."
-
-And then, too, there's the coming international angle.
-Electronic crime has never been easy to localize,
-to tie to a physical jurisdiction. And phone-phreaks
-and hackers loathe boundaries, they jump them whenever they can.
-The English. The Dutch. And the Germans, especially the ubiquitous
-Chaos Computer Club. The Australians. They've all learned phone-phreaking
-from America. It's a growth mischief industry. The multinational
-networks are global, but governments and the police simply aren't.
-Neither are the laws. Or the legal frameworks for citizen protection.
-
-One language is global, though--English. Phone phreaks speak English;
-it's their native tongue even if they're Germans. English may have started
-in England but now it's the Net language; it might as well be called "CNNese."
-
-Asians just aren't much into phone phreaking. They're the world masters
-at organized software piracy. The French aren't into phone-phreaking either.
-The French are into computerized industrial espionage.
-
-In the old days of the MIT righteous hackerdom, crashing systems
-didn't hurt anybody. Not all that much, anyway. Not permanently.
-Now the players are more venal. Now the consequences are worse.
-Hacking will begin killing people soon. Already there are methods
-of stacking calls onto 911 systems, annoying the police, and possibly
-causing the death of some poor soul calling in with a genuine emergency.
-Hackers in Amtrak computers, or air-traffic control computers, will kill
-somebody someday. Maybe a lot of people. Gail Thackeray expects it.
-
-And the viruses are getting nastier. The "Scud" virus is the latest one out.
-It wipes hard-disks.
-
-According to Thackeray, the idea that phone-phreaks are Robin Hoods is a fraud.
-They don't deserve this repute. Basically, they pick on the weak. AT&T now
-protects itself with the fearsome ANI (Automatic Number Identification)
-trace capability. When AT&T wised up and tightened security generally,
-the phreaks drifted into the Baby Bells. The Baby Bells lashed out in 1989
-and 1990, so the phreaks switched to smaller long-distance entrepreneurs.
-Today, they are moving into locally owned PBXes and voice-mail systems,
-which are full of security holes, dreadfully easy to hack. These victims
-aren't the moneybags Sheriff of Nottingham or Bad King John, but small groups
-of innocent people who find it hard to protect themselves, and who really
-suffer from these depredations. Phone phreaks pick on the weak. They do it
-for power. If it were legal, they wouldn't do it. They don't want service,
-or knowledge, they want the thrill of power-tripping. There's plenty of
-knowledge or service around if you're willing to pay. Phone phreaks don't pay,
-they steal. It's because it is illegal that it feels like power,
-that it gratifies their vanity.
-
-I leave Gail Thackeray with a handshake at the door of her office building--
-a vast International-Style office building downtown. The Sheriff's office
-is renting part of it. I get the vague impression that quite a lot of the
-building is empty--real estate crash.
-
-In a Phoenix sports apparel store, in a downtown mall, I meet
-the "Sun Devil" himself. He is the cartoon mascot of
-Arizona State University, whose football stadium, "Sundevil,"
-is near the local Secret Service HQ--hence the name Operation Sundevil.
-The Sun Devil himself is named "Sparky." Sparky the Sun Devil is maroon
-and bright yellow, the school colors. Sparky brandishes a three-tined
-yellow pitchfork. He has a small mustache, pointed ears, a barbed tail,
-and is dashing forward jabbing the air with the pitchfork,
-with an expression of devilish glee.
-
-Phoenix was the home of Operation Sundevil. The Legion of Doom
-ran a hacker bulletin board called "The Phoenix Project."
-An Australian hacker named "Phoenix" once burrowed through
-the Internet to attack Cliff Stoll, then bragged and boasted
-about it to The New York Times. This net of coincidence
-is both odd and meaningless.
-
-The headquarters of the Arizona Attorney General, Gail Thackeray's
-former workplace, is on 1275 Washington Avenue. Many of the downtown
-streets in Phoenix are named after prominent American presidents:
-Washington, Jefferson, Madison. . . .
-
-After dark, all the employees go home to their suburbs.
-Washington, Jefferson and Madison--what would be the
-Phoenix inner city, if there were an inner city in this
-sprawling automobile-bred town--become the haunts
-of transients and derelicts. The homeless. The sidewalks
-along Washington are lined with orange trees.
-Ripe fallen fruit lies scattered like croquet balls
-on the sidewalks and gutters. No one seems to be eating them.
-I try a fresh one. It tastes unbearably bitter.
-
-The Attorney General's office, built in 1981 during the
-Babbitt administration, is a long low two-story building
-of white cement and wall-sized sheets of curtain-glass.
-Behind each glass wall is a lawyer's office, quite open
-and visible to anyone strolling by. Across the street
-is a dour government building labelled simply ECONOMIC SECURITY,
-something that has not been in great supply in the American
-Southwest lately.
-
-The offices are about twelve feet square. They feature
-tall wooden cases full of red-spined lawbooks;
-Wang computer monitors; telephones; Post-it notes galore.
-Also framed law diplomas and a general excess of bad
-Western landscape art. Ansel Adams photos are a big favorite,
-perhaps to compensate for the dismal specter of the parking lot,
-two acres of striped black asphalt, which features gravel landscaping
-and some sickly-looking barrel cacti.
-
-It has grown dark. Gail Thackeray has told me that the people
-who work late here, are afraid of muggings in the parking lot.
-It seems cruelly ironic that a woman tracing electronic racketeers
-across the interstate labyrinth of Cyberspace should fear an assault
-by a homeless derelict in the parking lot of her own workplace.
-
-Perhaps this is less than coincidence. Perhaps these two seemingly
-disparate worlds are somehow generating one another. The poor and
-disenfranchised take to the streets, while the rich and computer-equipped,
-safe in their bedrooms, chatter over their modems. Quite often the derelicts
-kick the glass out and break in to the lawyers' offices, if they see something
-they need or want badly enough.
-
-I cross the parking lot to the street behind the Attorney General's office.
-A pair of young tramps are bedding down on flattened sheets of cardboard,
-under an alcove stretching over the sidewalk. One tramp wears a
-glitter-covered T-shirt reading "CALIFORNIA" in Coca-Cola cursive.
-His nose and cheeks look chafed and swollen; they glisten with
-what seems to be Vaseline. The other tramp has a ragged long-sleeved
-shirt and lank brown hair parted in the middle. They both wear blue jeans
-coated in grime. They are both drunk.
-
-"You guys crash here a lot?" I ask them.
-
-They look at me warily. I am wearing black jeans, a black pinstriped
-suit jacket and a black silk tie. I have odd shoes and a funny haircut.
-
-"It's our first time here," says the red-nosed tramp unconvincingly.
-There is a lot of cardboard stacked here. More than any two people could use.
-
-"We usually stay at the Vinnie's down the street," says the brown-haired tramp,
-puffing a Marlboro with a meditative air, as he sprawls with his head on
-a blue nylon backpack. "The Saint Vincent's."
-
-"You know who works in that building over there?" I ask, pointing.
-
-The brown-haired tramp shrugs. "Some kind of attorneys, it says."
-
-We urge one another to take it easy. I give them five bucks.
-
-A block down the street I meet a vigorous workman who is wheeling along
-some kind of industrial trolley; it has what appears to be a tank of
-propane on it.
-
-We make eye contact. We nod politely. I walk past him. "Hey!
-Excuse me sir!" he says.
-
-"Yes?" I say, stopping and turning.
-
-"Have you seen," the guy says rapidly, "a black guy, about 6'7",
-scars on both his cheeks like this--" he gestures-- "wears a
-black baseball cap on backwards, wandering around here anyplace?"
-
-"Sounds like I don't much WANT to meet him," I say.
-
-"He took my wallet," says my new acquaintance.
-"Took it this morning. Y'know, some people would be
-SCARED of a guy like that. But I'm not scared.
-I'm from Chicago. I'm gonna hunt him down.
-We do things like that in Chicago."
-
-"Yeah?"
-
-"I went to the cops and now he's got an APB out on his ass,"
-he says with satisfaction. "You run into him, you let me know."
-
-"Okay," I say. "What is your name, sir?"
-
-"Stanley. . . ."
-
-"And how can I reach you?"
-
-"Oh," Stanley says, in the same rapid voice,
-"you don't have to reach, uh, me.
-You can just call the cops. Go straight to the cops."
-He reaches into a pocket and pulls out a greasy piece of pasteboard.
-"See, here's my report on him."
-
-I look. The "report," the size of an index card, is labelled PRO-ACT:
-Phoenix Residents Opposing Active Crime Threat. . . . or is it
-Organized Against Crime Threat? In the darkening street it's hard
-to read. Some kind of vigilante group? Neighborhood watch?
-I feel very puzzled.
-
-"Are you a police officer, sir?"
-
-He smiles, seems very pleased by the question.
-
-"No," he says.
-
-"But you are a `Phoenix Resident?'"
-
-"Would you believe a homeless person," Stanley says.
-
-"Really? But what's with the. . . ." For the first time I take a close look
-at Stanley's trolley. It's a rubber-wheeled thing of industrial metal,
-but the device I had mistaken for a tank of propane is in fact a water-cooler.
-Stanley also has an Army duffel-bag, stuffed tight as a sausage with clothing
-or perhaps a tent, and, at the base of his trolley, a cardboard box and a
-battered leather briefcase.
-
-"I see," I say, quite at a loss. For the first time I notice that Stanley
-has a wallet. He has not lost his wallet at all. It is in his back pocket
-and chained to his belt. It's not a new wallet. It seems to have seen
-a lot of wear.
-
-"Well, you know how it is, brother," says Stanley.
-Now that I know that he is homeless--A POSSIBLE
-THREAT--my entire perception of him has changed
-in an instant. His speech, which once seemed just
-bright and enthusiastic, now seems to have a
-dangerous tang of mania. "I have to do this!"
-he assures me. "Track this guy down. . . .
-It's a thing I do. . . you know. . .to keep myself together!"
-He smiles, nods, lifts his trolley by its decaying rubber handgrips.
-
-"Gotta work together, y'know," Stanley booms, his face alight
-with cheerfulness, "the police can't do everything!"
-The gentlemen I met in my stroll in downtown Phoenix
-are the only computer illiterates in this book.
-To regard them as irrelevant, however, would be a grave mistake.
-
-As computerization spreads across society, the populace at large
-is subjected to wave after wave of future shock. But, as a
-necessary converse, the "computer community" itself is subjected
-to wave after wave of incoming computer illiterates.
-How will those currently enjoying America's digital bounty regard,
-and treat, all this teeming refuse yearning to breathe free?
-Will the electronic frontier be another Land of Opportunity--
-or an armed and monitored enclave, where the disenfranchised
-snuggle on their cardboard at the locked doors of our houses of justice?
-
-Some people just don't get along with computers. They can't read.
-They can't type. They just don't have it in their heads to master
-arcane instructions in wirebound manuals. Somewhere, the process
-of computerization of the populace will reach a limit. Some people--
-quite decent people maybe, who might have thrived in any other situation--
-will be left irretrievably outside the bounds. What's to be done with
-these people, in the bright new shiny electroworld? How will they
-be regarded, by the mouse-whizzing masters of cyberspace? With contempt?
-Indifference? Fear?
-
-In retrospect, it astonishes me to realize how quickly poor Stanley
-became a perceived threat. Surprise and fear are closely allied feelings.
-And the world of computing is full of surprises.
-
-I met one character in the streets of Phoenix whose role in this book
-is supremely and directly relevant. That personage was Stanley's giant
-thieving scarred phantom. This phantasm is everywhere in this book.
-He is the specter haunting cyberspace.
-
-Sometimes he's a maniac vandal ready to smash the phone system
-for no sane reason at all. Sometimes he's a fascist fed,
-coldly programming his mighty mainframes to destroy our Bill of Rights.
-Sometimes he's a telco bureaucrat, covertly conspiring to register all modems
-in the service of an Orwellian surveillance regime. Mostly, though,
-this fearsome phantom is a "hacker." He's strange, he doesn't belong,
-he's not authorized, he doesn't smell right, he's not keeping his proper place,
-he's not one of us. The focus of fear is the hacker, for much the same
-reasons that Stanley's fancied assailant is black.
-
-Stanley's demon can't go away, because he doesn't exist.
-Despite singleminded and tremendous effort, he can't be arrested,
-sued, jailed, or fired. The only constructive way to do ANYTHING
-about him is to learn more about Stanley himself. This learning process
-may be repellent, it may be ugly, it may involve grave elements of paranoiac
-confusion, but it's necessary. Knowing Stanley requires something more
-than class-crossing condescension. It requires more than steely
-legal objectivity. It requires human compassion and sympathy.
-
-To know Stanley is to know his demon. If you know the other guy's demon,
-then maybe you'll come to know some of your own. You'll be able to
-separate reality from illusion. And then you won't do your cause,
-and yourself, more harm than good. Like poor damned Stanley from Chicago did.
-
-#
-
-The Federal Computer Investigations Committee (FCIC) is the most important
-and influential organization in the realm of American computer-crime.
-Since the police of other countries have largely taken their computer-crime
-cues from American methods, the FCIC might well be called the most important
-computer crime group in the world.
-
-It is also, by federal standards, an organization of great unorthodoxy.
-State and local investigators mix with federal agents. Lawyers,
-financial auditors and computer-security programmers trade notes
-with street cops. Industry vendors and telco security people show up
-to explain their gadgetry and plead for protection and justice.
-Private investigators, think-tank experts and industry pundits throw in
-their two cents' worth. The FCIC is the antithesis of a formal bureaucracy.
-
-Members of the FCIC are obscurely proud of this fact; they recognize their
-group as aberrant, but are entirely convinced that this, for them,
-outright WEIRD behavior is nevertheless ABSOLUTELY NECESSARY
-to get their jobs done.
-
-FCIC regulars --from the Secret Service, the FBI, the IRS,
-the Department of Labor, the offices of federal attorneys,
-state police, the Air Force, from military intelligence--
-often attend meetings, held hither and thither across the country,
-at their own expense. The FCIC doesn't get grants. It doesn't
-charge membership fees. It doesn't have a boss. It has no headquarters--
-just a mail drop in Washington DC, at the Fraud Division of the Secret Service.
-It doesn't have a budget. It doesn't have schedules. It meets three times
-a year--sort of. Sometimes it issues publications, but the FCIC
-has no regular publisher, no treasurer, not even a secretary.
-There are no minutes of FCIC meetings. Non-federal people are considered
-"non-voting members," but there's not much in the way of elections.
-There are no badges, lapel pins or certificates of membership.
-Everyone is on a first-name basis. There are about forty of them.
-Nobody knows how many, exactly. People come, people go--
-sometimes people "go" formally but still hang around anyway.
-Nobody has ever exactly figured out what "membership" of this
-"Committee" actually entails.
-
-Strange as this may seem to some, to anyone familiar with the social world
-of computing, the "organization" of the FCIC is very recognizable.
-
-For years now, economists and management theorists have speculated
-that the tidal wave of the information revolution would destroy rigid,
-pyramidal bureaucracies, where everything is top-down and
-centrally controlled. Highly trained "employees" would take on
-much greater autonomy, being self-starting, and self-motivating,
-moving from place to place, task to task, with great speed and fluidity.
-"Ad-hocracy" would rule, with groups of people spontaneously knitting
-together across organizational lines, tackling the problem at hand,
-applying intense computer-aided expertise to it, and then vanishing
-whence they came.
-
-This is more or less what has actually happened in the world of
-federal computer investigation. With the conspicuous exception
-of the phone companies, which are after all over a hundred years old,
-practically EVERY organization that plays any important role in this book
-functions just like the FCIC. The Chicago Task Force, the Arizona
-Racketeering Unit, the Legion of Doom, the Phrack crowd, the
-Electronic Frontier Foundation--they ALL look and act like "tiger teams"
-or "user's groups." They are all electronic ad-hocracies leaping up
-spontaneously to attempt to meet a need.
-
-Some are police. Some are, by strict definition, criminals.
-Some are political interest-groups. But every single group
-has that same quality of apparent spontaneity--"Hey, gang!
-My uncle's got a barn--let's put on a show!"
-
-Every one of these groups is embarrassed by this "amateurism,"
-and, for the sake of their public image in a world of non-computer people,
-they all attempt to look as stern and formal and impressive as possible.
-These electronic frontier-dwellers resemble groups of nineteenth-century
-pioneers hankering after the respectability of statehood.
-There are however, two crucial differences in the historical experience
-of these "pioneers" of the nineteeth and twenty-first centuries.
-
-First, powerful information technology DOES play into the hands of small,
-fluid, loosely organized groups. There have always been "pioneers,"
-"hobbyists," "amateurs," "dilettantes," "volunteers," "movements,"
-"users' groups" and "blue-ribbon panels of experts" around.
-But a group of this kind--when technically equipped to ship
-huge amounts of specialized information, at lightning speed,
-to its members, to government, and to the press--is simply
-a different kind of animal. It's like the difference between
-an eel and an electric eel.
-
-The second crucial change is that American society is currently
-in a state approaching permanent technological revolution.
-In the world of computers particularly, it is practically impossible
-to EVER stop being a "pioneer," unless you either drop dead or
-deliberately jump off the bus. The scene has never slowed down
-enough to become well-institutionalized. And after twenty, thirty,
-forty years the "computer revolution" continues to spread,
-to permeate new corners of society. Anything that really works
-is already obsolete.
-
-If you spend your entire working life as a "pioneer," the word "pioneer"
-begins to lose its meaning. Your way of life looks less and less like
-an introduction to something else" more stable and organized,
-and more and more like JUST THE WAY THINGS ARE. A "permanent revolution"
-is really a contradiction in terms. If "turmoil" lasts long enough,
-it simply becomes A NEW KIND OF SOCIETY--still the same game of history,
-but new players, new rules.
-
-Apply this to the world of late twentieth-century law enforcement,
-and the implications are novel and puzzling indeed. Any bureaucratic
-rulebook you write about computer-crime will be flawed when you write it,
-and almost an antique by the time it sees print. The fluidity and fast
-reactions of the FCIC give them a great advantage in this regard,
-which explains their success. Even with the best will in the world
-(which it does not, in fact, possess) it is impossible for an organization
-the size of the U.S. Federal Bureau of Investigation to get up to speed
-on the theory and practice of computer crime. If they tried to train all
-their agents to do this, it would be SUICIDAL, as they would NEVER BE ABLE
-TO DO ANYTHING ELSE.
-
-The FBI does try to train its agents in the basics of electronic crime,
-at their base in Quantico, Virginia. And the Secret Service, along with
-many other law enforcement groups, runs quite successful and well-attended
-training courses on wire fraud, business crime, and computer intrusion
-at the Federal Law Enforcement Training Center (FLETC, pronounced "fletsy")
-in Glynco, Georgia. But the best efforts of these bureaucracies does not
-remove the absolute need for a "cutting-edge mess" like the FCIC.
-
-For you see--the members of FCIC ARE the trainers of the rest
-of law enforcement. Practically and literally speaking,
-they are the Glynco computer-crime faculty by another name.
-If the FCIC went over a cliff on a bus, the U.S. law enforcement
-community would be rendered deaf dumb and blind in the world
-of computer crime, and would swiftly feel a desperate need
-to reinvent them. And this is no time to go starting from scratch.
-
-On June 11, 1991, I once again arrived in Phoenix, Arizona,
-for the latest meeting of the Federal Computer Investigations Committee.
-This was more or less the twentieth meeting of this stellar group.
-The count was uncertain, since nobody could figure out whether to
-include the meetings of "the Colluquy," which is what the FCIC
-was called in the mid-1980s before it had even managed to obtain
-the dignity of its own acronym.
-
-Since my last visit to Arizona, in May, the local AzScam bribery scandal
-had resolved itself in a general muddle of humiliation. The Phoenix chief
-of police, whose agents had videotaped nine state legislators up to no good,
-had resigned his office in a tussle with the Phoenix city council over
-the propriety of his undercover operations.
-
-The Phoenix Chief could now join Gail Thackeray and eleven of her closest
-associates in the shared experience of politically motivated unemployment.
-As of June, resignations were still continuing at the Arizona Attorney
-General's office, which could be interpreted as either a New Broom
-Sweeping Clean or a Night of the Long Knives Part II, depending on
-your point of view.
-
-The meeting of FCIC was held at the Scottsdale Hilton Resort.
-Scottsdale is a wealthy suburb of Phoenix, known as "Scottsdull"
-to scoffing local trendies, but well-equipped with posh shopping-malls
-and manicured lawns, while conspicuously undersupplied with homeless derelicts.
-The Scottsdale Hilton Resort was a sprawling hotel in postmodern
-crypto-Southwestern style. It featured a "mission bell tower"
-plated in turquoise tile and vaguely resembling a Saudi minaret.
-
-Inside it was all barbarically striped Santa Fe Style decor.
-There was a health spa downstairs and a large oddly-shaped
-pool in the patio. A poolside umbrella-stand offered Ben and Jerry's
-politically correct Peace Pops.
-
-I registered as a member of FCIC, attaining a handy discount rate,
-then went in search of the Feds. Sure enough, at the back of the
-hotel grounds came the unmistakable sound of Gail Thackeray
-holding forth.
-
-Since I had also attended the Computers Freedom and Privacy conference
-(about which more later), this was the second time I had seen Thackeray
-in a group of her law enforcement colleagues. Once again I was struck
-by how simply pleased they seemed to see her. It was natural that she'd
-get SOME attention, as Gail was one of two women in a group of some thirty men;
-but there was a lot more to it than that.
-
-Gail Thackeray personifies the social glue of the FCIC. They could give
-a damn about her losing her job with the Attorney General. They were sorry
-about it, of course, but hell, they'd all lost jobs. If they were the kind
-of guys who liked steady boring jobs, they would never have gotten into
-computer work in the first place.
-
-I wandered into her circle and was immediately introduced to five strangers.
-The conditions of my visit at FCIC were reviewed. I would not quote
-anyone directly. I would not tie opinions expressed to the agencies
-of the attendees. I would not (a purely hypothetical example)
-report the conversation of a guy from the Secret Service talking
-quite civilly to a guy from the FBI, as these two agencies NEVER
-talk to each other, and the IRS (also present, also hypothetical)
-NEVER TALKS TO ANYBODY.
-
-Worse yet, I was forbidden to attend the first conference. And I didn't.
-I have no idea what the FCIC was up to behind closed doors that afternoon.
-I rather suspect that they were engaging in a frank and thorough confession
-of their errors, goof-ups and blunders, as this has been a feature of every
-FCIC meeting since their legendary Memphis beer-bust of 1986. Perhaps the
-single greatest attraction of FCIC is that it is a place where you can go,
-let your hair down, and completely level with people who actually comprehend
-what you are talking about. Not only do they understand you, but they
-REALLY PAY ATTENTION, they are GRATEFUL FOR YOUR INSIGHTS, and they
-FORGIVE YOU, which in nine cases out of ten is something even your
-boss can't do, because as soon as you start talking "ROM," "BBS,"
-or "T-1 trunk," his eyes glaze over.
-
-I had nothing much to do that afternoon. The FCIC were beavering away
-in their conference room. Doors were firmly closed, windows too dark
-to peer through. I wondered what a real hacker, a computer intruder,
-would do at a meeting like this.
-
-The answer came at once. He would "trash" the place. Not reduce the place
-to trash in some orgy of vandalism; that's not the use of the term in the
-hacker milieu. No, he would quietly EMPTY THE TRASH BASKETS and silently
-raid any valuable data indiscreetly thrown away.
-
-Journalists have been known to do this. (Journalists hunting information
-have been known to do almost every single unethical thing that hackers
-have ever done. They also throw in a few awful techniques all their own.)
-The legality of `trashing' is somewhat dubious but it is not in fact
-flagrantly illegal. It was, however, absurd to contemplate trashing the FCIC.
-These people knew all about trashing. I wouldn't last fifteen seconds.
-
-The idea sounded interesting, though. I'd been hearing a lot about
-the practice lately. On the spur of the moment, I decided I would try
-trashing the office ACROSS THE HALL from the FCIC, an area which had
-nothing to do with the investigators.
-
-The office was tiny; six chairs, a table. . . . Nevertheless, it was open,
-so I dug around in its plastic trash can.
-
-To my utter astonishment, I came up with the torn scraps of a SPRINT
-long-distance phone bill. More digging produced a bank statement
-and the scraps of a hand-written letter, along with gum, cigarette ashes,
-candy wrappers and a day-old-issue of USA TODAY.
-
-The trash went back in its receptacle while the scraps of data went into
-my travel bag. I detoured through the hotel souvenir shop for some
-Scotch tape and went up to my room.
-
-Coincidence or not, it was quite true. Some poor soul had, in fact,
-thrown a SPRINT bill into the hotel's trash. Date May 1991,
-total amount due: $252.36. Not a business phone, either,
-but a residential bill, in the name of someone called Evelyn
-(not her real name). Evelyn's records showed a ## PAST DUE BILL ##!
-Here was her nine-digit account ID. Here was a stern computer-printed warning:
-
-"TREAT YOUR FONCARD AS YOU WOULD ANY CREDIT CARD. TO SECURE AGAINST FRAUD,
-NEVER GIVE YOUR FONCARD NUMBER OVER THE PHONE UNLESS YOU INITIATED THE CALL.
-IF YOU RECEIVE SUSPICIOUS CALLS PLEASE NOTIFY CUSTOMER SERVICE IMMEDIATELY!"
-
-I examined my watch. Still plenty of time left for the FCIC to carry on.
-I sorted out the scraps of Evelyn's SPRINT bill and re-assembled them with
-fresh Scotch tape. Here was her ten-digit FONCARD number. Didn't seem
-to have the ID number necessary to cause real fraud trouble.
-
-I did, however, have Evelyn's home phone number. And the phone numbers
-for a whole crowd of Evelyn's long-distance friends and acquaintances.
-In San Diego, Folsom, Redondo, Las Vegas, La Jolla, Topeka, and Northampton
-Massachusetts. Even somebody in Australia!
-
-I examined other documents. Here was a bank statement. It was Evelyn's
-IRA account down at a bank in San Mateo California (total balance $1877.20).
-Here was a charge-card bill for $382.64. She was paying it off bit by bit.
-
-Driven by motives that were completely unethical and prurient,
-I now examined the handwritten notes. They had been torn fairly
-thoroughly, so much so that it took me almost an entire five minutes
-to reassemble them.
-
-They were drafts of a love letter. They had been written on
-the lined stationery of Evelyn's employer, a biomedical company.
-Probably written at work when she should have been doing something else.
-
-"Dear Bob," (not his real name) "I guess in everyone's life there comes
-a time when hard decisions have to be made, and this is a difficult one
-for me--very upsetting. Since you haven't called me, and I don't understand
-why, I can only surmise it's because you don't want to. I thought I would
-have heard from you Friday. I did have a few unusual problems with my phone
-and possibly you tried, I hope so.
-
-"Robert, you asked me to `let go'. . . ."
-
-The first note ended. UNUSUAL PROBLEMS WITH HER PHONE?
-I looked swiftly at the next note.
-
-"Bob, not hearing from you for the whole weekend has left me very perplexed. . . ."
-
-Next draft.
-
-"Dear Bob, there is so much I don't understand right now, and I wish I did.
-I wish I could talk to you, but for some unknown reason you have elected not
-to call--this is so difficult for me to understand. . . ."
-
-She tried again.
-
-"Bob, Since I have always held you in such high esteem, I had every hope that
-we could remain good friends, but now one essential ingredient is missing--
-respect. Your ability to discard people when their purpose is served is
-appalling to me. The kindest thing you could do for me now is to leave me
-alone. You are no longer welcome in my heart or home. . . ."
-
-Try again.
-
-"Bob, I wrote a very factual note to you to say how much respect I had lost
-for you, by the way you treat people, me in particular, so uncaring and cold.
-The kindest thing you can do for me is to leave me alone entirely,
-as you are no longer welcome in my heart or home. I would appreciate it
-if you could retire your debt to me as soon as possible--I wish no link
-to you in any way. Sincerely, Evelyn."
-
-Good heavens, I thought, the bastard actually owes her money!
-I turned to the next page.
-
-"Bob: very simple. GOODBYE! No more mind games--no more fascination--
-no more coldness--no more respect for you! It's over--Finis. Evie"
-
-There were two versions of the final brushoff letter, but they read about
-the same. Maybe she hadn't sent it. The final item in my illicit and
-shameful booty was an envelope addressed to "Bob" at his home address,
-but it had no stamp on it and it hadn't been mailed.
-
-Maybe she'd just been blowing off steam because her rascal boyfriend
-had neglected to call her one weekend. Big deal. Maybe they'd kissed
-and made up, maybe she and Bob were down at Pop's Chocolate Shop now,
-sharing a malted. Sure.
-
-Easy to find out. All I had to do was call Evelyn up. With a half-clever
-story and enough brass-plated gall I could probably trick the truth out of her.
-Phone-phreaks and hackers deceive people over the phone all the time.
-It's called "social engineering." Social engineering is a very common practice
-in the underground, and almost magically effective. Human beings are almost
-always the weakest link in computer security. The simplest way to learn
-Things You Are Not Meant To Know is simply to call up and exploit the
-knowledgeable people. With social engineering, you use the bits of specialized
-knowledge you already have as a key, to manipulate people into believing
-that you are legitimate. You can then coax, flatter, or frighten them into
-revealing almost anything you want to know. Deceiving people (especially
-over the phone) is easy and fun. Exploiting their gullibility is very
-gratifying; it makes you feel very superior to them.
-
-If I'd been a malicious hacker on a trashing raid, I would now have Evelyn
-very much in my power. Given all this inside data, it wouldn't take much
-effort at all to invent a convincing lie. If I were ruthless enough,
-and jaded enough, and clever enough, this momentary indiscretion of hers--
-maybe committed in tears, who knows--could cause her a whole world of
-confusion and grief.
-
-I didn't even have to have a MALICIOUS motive. Maybe I'd be "on her side,"
-and call up Bob instead, and anonymously threaten to break both his kneecaps
-if he didn't take Evelyn out for a steak dinner pronto. It was still
-profoundly NONE OF MY BUSINESS. To have gotten this knowledge at all
-was a sordid act and to use it would be to inflict a sordid injury.
-
-To do all these awful things would require exactly zero high-tech expertise.
-All it would take was the willingness to do it and a certain amount
-of bent imagination.
-
-I went back downstairs. The hard-working FCIC, who had labored forty-five
-minutes over their schedule, were through for the day, and adjourned to the
-hotel bar. We all had a beer.
-
-I had a chat with a guy about "Isis," or rather IACIS,
-the International Association of Computer Investigation Specialists.
-They're into "computer forensics," the techniques of picking computer-
-systems apart without destroying vital evidence. IACIS, currently run
-out of Oregon, is comprised of investigators in the U.S., Canada, Taiwan
-and Ireland. "Taiwan and Ireland?" I said. Are TAIWAN and IRELAND
-really in the forefront of this stuff? Well not exactly, my informant
-admitted. They just happen to have been the first ones to have caught
-on by word of mouth. Still, the international angle counts, because this
-is obviously an international problem. Phone-lines go everywhere.
-
-There was a Mountie here from the Royal Canadian Mounted Police.
-He seemed to be having quite a good time. Nobody had flung this
-Canadian out because he might pose a foreign security risk.
-These are cyberspace cops. They still worry a lot about "jurisdictions,"
-but mere geography is the least of their troubles.
-
-NASA had failed to show. NASA suffers a lot from computer intrusions,
-in particular from Australian raiders and a well-trumpeted Chaos
-Computer Club case, and in 1990 there was a brief press flurry
-when it was revealed that one of NASA's Houston branch-exchanges
-had been systematically ripped off by a gang of phone-phreaks.
-But the NASA guys had had their funding cut. They were stripping everything.
-
-Air Force OSI, its Office of Special Investigations, is the ONLY federal
-entity dedicated full-time to computer security. They'd been expected
-to show up in force, but some of them had cancelled--a Pentagon budget pinch.
-
-As the empties piled up, the guys began joshing around and telling war-stories.
-"These are cops," Thackeray said tolerantly. "If they're not talking shop
-they talk about women and beer."
-
-I heard the story about the guy who, asked for "a copy" of a computer disk,
-PHOTOCOPIED THE LABEL ON IT. He put the floppy disk onto the glass plate
-of a photocopier. The blast of static when the copier worked completely
-erased all the real information on the disk.
-
-Some other poor souls threw a whole bag of confiscated diskettes
-into the squad-car trunk next to the police radio. The powerful radio
-signal blasted them, too.
-
-We heard a bit about Dave Geneson, the first computer prosecutor,
-a mainframe-runner in Dade County, turned lawyer. Dave Geneson
-was one guy who had hit the ground running, a signal virtue
-in making the transition to computer-crime. It was generally
-agreed that it was easier to learn the world of computers first,
-then police or prosecutorial work. You could take certain computer
-people and train 'em to successful police work--but of course they
-had to have the COP MENTALITY. They had to have street smarts.
-Patience. Persistence. And discretion. You've got to make sure
-they're not hot-shots, show-offs, "cowboys."
-
-Most of the folks in the bar had backgrounds in military intelligence,
-or drugs, or homicide. It was rudely opined that "military intelligence"
-was a contradiction in terms, while even the grisly world of homicide
-was considered cleaner than drug enforcement. One guy had been 'way
-undercover doing dope-work in Europe for four years straight.
-"I'm almost recovered now," he said deadpan, with the acid black humor
-that is pure cop. "Hey, now I can say FUCKER without putting MOTHER
-in front of it."
-
-"In the cop world," another guy said earnestly, "everything is good and bad,
-black and white. In the computer world everything is gray."
-
-One guy--a founder of the FCIC, who'd been with the group
-since it was just the Colluquy--described his own introduction
-to the field. He'd been a Washington DC homicide guy called in
-on a "hacker" case. From the word "hacker," he naturally assumed
-he was on the trail of a knife-wielding marauder, and went to the
-computer center expecting blood and a body. When he finally figured
-out what was happening there (after loudly demanding, in vain,
-that the programmers "speak English"), he called headquarters
-and told them he was clueless about computers. They told him nobody
-else knew diddly either, and to get the hell back to work.
-
-So, he said, he had proceeded by comparisons. By analogy. By metaphor.
-"Somebody broke in to your computer, huh?" Breaking and entering;
-I can understand that. How'd he get in? "Over the phone-lines."
-Harassing phone-calls, I can understand that! What we need here
-is a tap and a trace!
-
-It worked. It was better than nothing. And it worked a lot faster
-when he got hold of another cop who'd done something similar.
-And then the two of them got another, and another, and pretty soon
-the Colluquy was a happening thing. It helped a lot that everybody
-seemed to know Carlton Fitzpatrick, the data-processing trainer in Glynco.
-
-The ice broke big-time in Memphis in '86. The Colluquy had attracted
-a bunch of new guys--Secret Service, FBI, military, other feds, heavy guys.
-Nobody wanted to tell anybody anything. They suspected that if word got back
-to the home office they'd all be fired. They passed an uncomfortably
-guarded afternoon.
-
-The formalities got them nowhere. But after the formal session was over,
-the organizers brought in a case of beer. As soon as the participants
-knocked it off with the bureaucratic ranks and turf-fighting, everything
-changed. "I bared my soul," one veteran reminisced proudly. By nightfall
-they were building pyramids of empty beer-cans and doing everything
-but composing a team fight song.
-
-FCIC were not the only computer-crime people around. There was DATTA
-(District Attorneys' Technology Theft Association), though they mostly
-specialized in chip theft, intellectual property, and black-market cases.
-There was HTCIA (High Tech Computer Investigators Association),
-also out in Silicon Valley, a year older than FCIC and featuring
-brilliant people like Donald Ingraham. There was LEETAC
-(Law Enforcement Electronic Technology Assistance Committee)
-in Florida, and computer-crime units in Illinois and Maryland
-and Texas and Ohio and Colorado and Pennsylvania. But these were
-local groups. FCIC were the first to really network nationally
-and on a federal level.
-
-FCIC people live on the phone lines. Not on bulletin board systems--
-they know very well what boards are, and they know that boards aren't secure.
-Everyone in the FCIC has a voice-phone bill like you wouldn't believe.
-FCIC people have been tight with the telco people for a long time.
-Telephone cyberspace is their native habitat.
-
-FCIC has three basic sub-tribes: the trainers, the security people,
-and the investigators. That's why it's called an "Investigations
-Committee" with no mention of the term "computer-crime"--the dreaded
-"C-word." FCIC, officially, is "an association of agencies rather
-than individuals;" unofficially, this field is small enough that
-the influence of individuals and individual expertise is paramount.
-Attendance is by invitation only, and most everyone in FCIC considers
-himself a prophet without honor in his own house.
-
-Again and again I heard this, with different terms but identical
-sentiments. "I'd been sitting in the wilderness talking to myself."
-"I was totally isolated." "I was desperate." "FCIC is the best
-thing there is about computer crime in America." "FCIC is what
-really works." "This is where you hear real people telling you
-what's really happening out there, not just lawyers picking nits."
-"We taught each other everything we knew."
-
-The sincerity of these statements convinces me that this is true.
-FCIC is the real thing and it is invaluable. It's also very sharply
-at odds with the rest of the traditions and power structure
-in American law enforcement. There probably hasn't been anything
-around as loose and go-getting as the FCIC since the start of the
-U.S. Secret Service in the 1860s. FCIC people are living like
-twenty-first-century people in a twentieth-century environment,
-and while there's a great deal to be said for that, there's also
-a great deal to be said against it, and those against it happen
-to control the budgets.
-
-I listened to two FCIC guys from Jersey compare life histories.
-One of them had been a biker in a fairly heavy-duty gang in the 1960s.
-"Oh, did you know so-and-so?" said the other guy from Jersey.
-"Big guy, heavyset?"
-
-"Yeah, I knew him."
-
-"Yeah, he was one of ours. He was our plant in the gang."
-
-"Really? Wow! Yeah, I knew him. Helluva guy."
-
-Thackeray reminisced at length about being tear-gassed blind
-in the November 1969 antiwar protests in Washington Circle,
-covering them for her college paper. "Oh yeah, I was there,"
-said another cop. "Glad to hear that tear gas hit somethin'.
-Haw haw haw." He'd been so blind himself, he confessed,
-that later that day he'd arrested a small tree.
-
-FCIC are an odd group, sifted out by coincidence and necessity,
-and turned into a new kind of cop. There are a lot of specialized
-cops in the world--your bunco guys, your drug guys, your tax guys,
-but the only group that matches FCIC for sheer isolation are probably
-the child-pornography people. Because they both deal with conspirators
-who are desperate to exchange forbidden data and also desperate to hide;
-and because nobody else in law enforcement even wants to hear about it.
-
-FCIC people tend to change jobs a lot. They tend not to get the equipment
-and training they want and need. And they tend to get sued quite often.
-
-As the night wore on and a band set up in the bar, the talk grew darker.
-Nothing ever gets done in government, someone opined, until there's
-a DISASTER. Computing disasters are awful, but there's no denying
-that they greatly help the credibility of FCIC people. The Internet Worm,
-for instance. "For years we'd been warning about that--but it's nothing
-compared to what's coming." They expect horrors, these people.
-They know that nothing will really get done until there is a horror.
-
-#
-
-Next day we heard an extensive briefing from a guy who'd been a computer cop,
-gotten into hot water with an Arizona city council, and now installed
-computer networks for a living (at a considerable rise in pay).
-He talked about pulling fiber-optic networks apart.
-
-Even a single computer, with enough peripherals, is a literal
-"network"--a bunch of machines all cabled together, generally
-with a complexity that puts stereo units to shame. FCIC people
-invent and publicize methods of seizing computers and maintaining
-their evidence. Simple things, sometimes, but vital rules of thumb
-for street cops, who nowadays often stumble across a busy computer
-in the midst of a drug investigation or a white-collar bust.
-For instance: Photograph the system before you touch it.
-Label the ends of all the cables before you detach anything.
-"Park" the heads on the disk drives before you move them.
-Get the diskettes. Don't put the diskettes in magnetic fields.
-Don't write on diskettes with ballpoint pens. Get the manuals.
-Get the printouts. Get the handwritten notes. Copy data before
-you look at it, and then examine the copy instead of the original.
-
-Now our lecturer distributed copied diagrams of a typical LAN
-or "Local Area Network", which happened to be out of Connecticut.
-ONE HUNDRED AND FIFTY-NINE desktop computers, each with its own
-peripherals. Three "file servers." Five "star couplers"
-each with thirty-two ports. One sixteen-port coupler
-off in the corner office. All these machines talking to each other,
-distributing electronic mail, distributing software, distributing,
-quite possibly, criminal evidence. All linked by high-capacity
-fiber-optic cable. A bad guy--cops talk a about "bad guys"
---might be lurking on PC #47 lot or #123 and distributing
-his ill doings onto some dupe's "personal" machine in
-another office--or another floor--or, quite possibly,
-two or three miles away! Or, conceivably, the evidence might
-be "data-striped"--split up into meaningless slivers stored,
-one by one, on a whole crowd of different disk drives.
-
-The lecturer challenged us for solutions. I for one was utterly clueless.
-As far as I could figure, the Cossacks were at the gate; there were probably
-more disks in this single building than were seized during the entirety
-of Operation Sundevil.
-
-"Inside informant," somebody said. Right. There's always the human angle,
-something easy to forget when contemplating the arcane recesses of high
-technology. Cops are skilled at getting people to talk, and computer people,
-given a chair and some sustained attention, will talk about their computers
-till their throats go raw. There's a case on record of a single question--
-"How'd you do it?"--eliciting a forty-five-minute videotaped confession
-from a computer criminal who not only completely incriminated himself
-but drew helpful diagrams.
-
-Computer people talk. Hackers BRAG. Phone-phreaks
-talk PATHOLOGICALLY--why else are they stealing phone-codes,
-if not to natter for ten hours straight to their friends
-on an opposite seaboard? Computer-literate people do
-in fact possess an arsenal of nifty gadgets and techniques
-that would allow them to conceal all kinds of exotic skullduggery,
-and if they could only SHUT UP about it, they could probably
-get away with all manner of amazing information-crimes.
-But that's just not how it works--or at least,
-that's not how it's worked SO FAR.
-
-Most every phone-phreak ever busted has swiftly implicated his mentors,
-his disciples, and his friends. Most every white-collar computer-criminal,
-smugly convinced that his clever scheme is bulletproof, swiftly learns
-otherwise when, for the first time in his life, an actual no-kidding
-policeman leans over, grabs the front of his shirt, looks him right
-in the eye and says: "All right, ASSHOLE--you and me are going downtown!"
-All the hardware in the world will not insulate your nerves from
-these actual real-life sensations of terror and guilt.
-
-Cops know ways to get from point A to point Z without thumbing
-through every letter in some smart-ass bad-guy's alphabet.
-Cops know how to cut to the chase. Cops know a lot of things
-other people don't know.
-
-Hackers know a lot of things other people don't know, too.
-Hackers know, for instance, how to sneak into your computer
-through the phone-lines. But cops can show up RIGHT ON YOUR DOORSTEP
-and carry off YOU and your computer in separate steel boxes.
-A cop interested in hackers can grab them and grill them.
-A hacker interested in cops has to depend on hearsay,
-underground legends, and what cops are willing to publicly reveal.
-And the Secret Service didn't get named "the SECRET Service"
-because they blab a lot.
-
-Some people, our lecturer informed us, were under the mistaken
-impression that it was "impossible" to tap a fiber-optic line.
-Well, he announced, he and his son had just whipped up a
-fiber-optic tap in his workshop at home. He passed it around
-the audience, along with a circuit-covered LAN plug-in card
-so we'd all recognize one if we saw it on a case. We all had a look.
-
-The tap was a classic "Goofy Prototype"--a thumb-length rounded
-metal cylinder with a pair of plastic brackets on it.
-From one end dangled three thin black cables, each of which ended
-in a tiny black plastic cap. When you plucked the safety-cap
-off the end of a cable, you could see the glass fiber--
-no thicker than a pinhole.
-
-Our lecturer informed us that the metal cylinder was a
-"wavelength division multiplexer." Apparently, what one did
-was to cut the fiber-optic cable, insert two of the legs into
-the cut to complete the network again, and then read any passing data
-on the line by hooking up the third leg to some kind of monitor.
-Sounded simple enough. I wondered why nobody had thought of it before.
-I also wondered whether this guy's son back at the workshop had any
-teenage friends.
-
-We had a break. The guy sitting next to me was wearing a giveaway
-baseball cap advertising the Uzi submachine gun. We had a desultory chat
-about the merits of Uzis. Long a favorite of the Secret Service,
-it seems Uzis went out of fashion with the advent of the Persian Gulf War,
-our Arab allies taking some offense at Americans toting Israeli weapons.
-Besides, I was informed by another expert, Uzis jam. The equivalent weapon
-of choice today is the Heckler & Koch, manufactured in Germany.
-
-The guy with the Uzi cap was a forensic photographer. He also did a lot
-of photographic surveillance work in computer crime cases. He used to,
-that is, until the firings in Phoenix. He was now a private investigator and,
-with his wife, ran a photography salon specializing in weddings and portrait
-photos. At--one must repeat--a considerable rise in income.
-
-He was still FCIC. If you were FCIC, and you needed to talk
-to an expert about forensic photography, well, there he was,
-willing and able. If he hadn't shown up, people would have missed him.
-
-Our lecturer had raised the point that preliminary investigation
-of a computer system is vital before any seizure is undertaken.
-It's vital to understand how many machines are in there, what kinds
-there are, what kind of operating system they use, how many people
-use them, where the actual data itself is stored. To simply barge into
-an office demanding "all the computers" is a recipe for swift disaster.
-
-This entails some discreet inquiries beforehand. In fact, what it
-entails is basically undercover work. An intelligence operation.
-SPYING, not to put too fine a point on it.
-
-In a chat after the lecture, I asked an attendee whether "trashing" might work.
-
-I received a swift briefing on the theory and practice of "trash covers."
-Police "trash covers," like "mail covers" or like wiretaps, require the
-agreement of a judge. This obtained, the "trashing" work of cops is just
-like that of hackers, only more so and much better organized. So much so,
-I was informed, that mobsters in Phoenix make extensive use of locked
-garbage cans picked up by a specialty high-security trash company.
-
-In one case, a tiger team of Arizona cops had trashed a local residence
-for four months. Every week they showed up on the municipal garbage truck,
-disguised as garbagemen, and carried the contents of the suspect cans off
-to a shade tree, where they combed through the garbage--a messy task,
-especially considering that one of the occupants was undergoing
-kidney dialysis. All useful documents were cleaned, dried and examined.
-A discarded typewriter-ribbon was an especially valuable source of data,
-as its long one-strike ribbon of film contained the contents of every
-letter mailed out of the house. The letters were neatly retyped by
-a police secretary equipped with a large desk-mounted magnifying glass.
-
-There is something weirdly disquieting about the whole subject of
-"trashing"-- an unsuspected and indeed rather disgusting mode of
-deep personal vulnerability. Things that we pass by every day,
-that we take utterly for granted, can be exploited with so little work.
-Once discovered, the knowledge of these vulnerabilities tend to spread.
-
-Take the lowly subject of MANHOLE COVERS. The humble manhole cover
-reproduces many of the dilemmas of computer-security in miniature.
-Manhole covers are, of course, technological artifacts, access-points
-to our buried urban infrastructure. To the vast majority of us,
-manhole covers are invisible. They are also vulnerable. For many years now,
-the Secret Service has made a point of caulking manhole covers along all routes
-of the Presidential motorcade. This is, of course, to deter terrorists from
-leaping out of underground ambush or, more likely, planting remote-control
-car-smashing bombs beneath the street.
-
-Lately, manhole covers have seen more and more criminal exploitation,
-especially in New York City. Recently, a telco in New York City
-discovered that a cable television service had been sneaking into
-telco manholes and installing cable service alongside the phone-lines--
-WITHOUT PAYING ROYALTIES. New York companies have also suffered a
-general plague of (a) underground copper cable theft; (b) dumping of garbage,
-including toxic waste, and (c) hasty dumping of murder victims.
-
-Industry complaints reached the ears of an innovative New England
-industrial-security company, and the result was a new product known
-as "the Intimidator," a thick titanium-steel bolt with a precisely machined
-head that requires a special device to unscrew. All these "keys" have registered
-serial numbers kept on file with the manufacturer. There are now some
-thousands of these "Intimidator" bolts being sunk into American pavements
-wherever our President passes, like some macabre parody of strewn roses.
-They are also spreading as fast as steel dandelions around US military bases
-and many centers of private industry.
-
-Quite likely it has never occurred to you to peer under a manhole cover,
-perhaps climb down and walk around down there with a flashlight, just to see
-what it's like. Formally speaking, this might be trespassing, but if you
-didn't hurt anything, and didn't make an absolute habit of it, nobody would
-really care. The freedom to sneak under manholes was likely a freedom
-you never intended to exercise.
-
-You now are rather less likely to have that freedom at all.
-You may never even have missed it until you read about it here,
-but if you're in New York City it's gone, and elsewhere it's likely going.
-This is one of the things that crime, and the reaction to
-crime, does to us.
-
-The tenor of the meeting now changed as the Electronic Frontier Foundation
-arrived. The EFF, whose personnel and history will be examined in detail
-in the next chapter, are a pioneering civil liberties group who arose in
-direct response to the Hacker Crackdown of 1990.
-
-Now Mitchell Kapor, the Foundation's president, and Michael Godwin,
-its chief attorney, were confronting federal law enforcement MANO A MANO
-for the first time ever. Ever alert to the manifold uses of publicity,
-Mitch Kapor and Mike Godwin had brought their own journalist in tow:
-Robert Draper, from Austin, whose recent well-received book about
-ROLLING STONE magazine was still on the stands. Draper was on assignment
-for TEXAS MONTHLY.
-
-The Steve Jackson/EFF civil lawsuit against the Chicago Computer Fraud
-and Abuse Task Force was a matter of considerable regional interest in Texas.
-There were now two Austinite journalists here on the case. In fact,
-counting Godwin (a former Austinite and former journalist) there were
-three of us. Lunch was like Old Home Week.
-
-Later, I took Draper up to my hotel room. We had a long frank talk
-about the case, networking earnestly like a miniature freelance-journo
-version of the FCIC: privately confessing the numerous blunders
-of journalists covering the story, and trying hard to figure out
-who was who and what the hell was really going on out there.
-I showed Draper everything I had dug out of the Hilton trashcan.
-We pondered the ethics of "trashing" for a while, and agreed
-that they were dismal. We also agreed that finding a SPRINT
-bill on your first time out was a heck of a coincidence.
-
-First I'd "trashed"--and now, mere hours later, I'd bragged to someone else.
-Having entered the lifestyle of hackerdom, I was now, unsurprisingly,
-following its logic. Having discovered something remarkable through
-a surreptitious action, I of course HAD to "brag," and to drag the passing
-Draper into my iniquities. I felt I needed a witness. Otherwise nobody
-would have believed what I'd discovered. . . .
-
-Back at the meeting, Thackeray cordially, if rather tentatively,
-introduced Kapor and Godwin to her colleagues. Papers were distributed.
-Kapor took center stage. The brilliant Bostonian high-tech entrepreneur,
-normally the hawk in his own administration and quite an effective
-public speaker, seemed visibly nervous, and frankly admitted as much.
-He began by saying he consided computer-intrusion to be morally wrong,
-and that the EFF was not a "hacker defense fund," despite what had appeared
-in print. Kapor chatted a bit about the basic motivations of his group,
-emphasizing their good faith and willingness to listen and seek common ground
-with law enforcement--when, er, possible.
-
-Then, at Godwin's urging, Kapor suddenly remarked that EFF's own Internet
-machine had been "hacked" recently, and that EFF did not consider
-this incident amusing.
-
-After this surprising confession, things began to loosen up
-quite rapidly. Soon Kapor was fielding questions, parrying objections,
-challenging definitions, and juggling paradigms with something akin
-to his usual gusto.
-
-Kapor seemed to score quite an effect with his shrewd and skeptical analysis
-of the merits of telco "Caller-ID" services. (On this topic, FCIC and EFF
-have never been at loggerheads, and have no particular established earthworks
-to defend.) Caller-ID has generally been promoted as a privacy service
-for consumers, a presentation Kapor described as a "smokescreen,"
-the real point of Caller-ID being to ALLOW CORPORATE CUSTOMERS TO BUILD
-EXTENSIVE COMMERCIAL DATABASES ON EVERYBODY WHO PHONES OR FAXES THEM.
-Clearly, few people in the room had considered this possibility,
-except perhaps for two late-arrivals from US WEST RBOC security,
-who chuckled nervously.
-
-Mike Godwin then made an extensive presentation on
-"Civil Liberties Implications of Computer Searches and Seizures."
-Now, at last, we were getting to the real nitty-gritty here,
-real political horse-trading. The audience listened with close
-attention, angry mutters rising occasionally: "He's trying to
-teach us our jobs!" "We've been thinking about this for years!
-We think about these issues every day!" "If I didn't seize the works,
-I'd be sued by the guy's victims!" "I'm violating the law if I leave
-ten thousand disks full of illegal PIRATED SOFTWARE and STOLEN CODES!"
-"It's our job to make sure people don't trash the Constitution--
-we're the DEFENDERS of the Constitution!" "We seize stuff when
-we know it will be forfeited anyway as restitution for the victim!"
-
-"If it's forfeitable, then don't get a search warrant, get a
-forfeiture warrant," Godwin suggested coolly. He further remarked
-that most suspects in computer crime don't WANT to see their computers
-vanish out the door, headed God knew where, for who knows how long.
-They might not mind a search, even an extensive search, but they want
-their machines searched on-site.
-
-"Are they gonna feed us?" somebody asked sourly.
-
-"How about if you take copies of the data?" Godwin parried.
-
-"That'll never stand up in court."
-
-"Okay, you make copies, give THEM the copies, and take the originals."
-
-Hmmm.
-
-Godwin championed bulletin-board systems as repositories of First Amendment
-protected free speech. He complained that federal computer-crime training
-manuals gave boards a bad press, suggesting that they are hotbeds of crime
-haunted by pedophiles and crooks, whereas the vast majority of the nation's
-thousands of boards are completely innocuous, and nowhere near so
-romantically suspicious.
-
-People who run boards violently resent it when their systems are seized,
-and their dozens (or hundreds) of users look on in abject horror.
-Their rights of free expression are cut short. Their right to associate
-with other people is infringed. And their privacy is violated as their
-private electronic mail becomes police property.
-
-Not a soul spoke up to defend the practice of seizing boards.
-The issue passed in chastened silence. Legal principles aside--
-(and those principles cannot be settled without laws passed or
-court precedents)--seizing bulletin boards has become public-relations
-poison for American computer police.
-
-And anyway, it's not entirely necessary. If you're a cop, you can get 'most
-everything you need from a pirate board, just by using an inside informant.
-Plenty of vigilantes--well, CONCERNED CITIZENS--will inform police the moment
-they see a pirate board hit their area (and will tell the police all about it,
-in such technical detail, actually, that you kinda wish they'd shut up).
-They will happily supply police with extensive downloads or printouts.
-It's IMPOSSIBLE to keep this fluid electronic information out of the
-hands of police.
-
-Some people in the electronic community become enraged at the prospect
-of cops "monitoring" bulletin boards. This does have touchy aspects,
-as Secret Service people in particular examine bulletin boards with
-some regularity. But to expect electronic police to be deaf dumb
-and blind in regard to this particular medium rather flies in the face
-of common sense. Police watch television, listen to radio, read newspapers
-and magazines; why should the new medium of boards be different?
-Cops can exercise the same access to electronic information
-as everybody else. As we have seen, quite a few computer
-police maintain THEIR OWN bulletin boards, including anti-hacker
-"sting" boards, which have generally proven quite effective.
-
-As a final clincher, their Mountie friends in Canada (and colleagues
-in Ireland and Taiwan) don't have First Amendment or American
-constitutional restrictions, but they do have phone lines,
-and can call any bulletin board in America whenever they please.
-The same technological determinants that play into the hands of hackers,
-phone phreaks and software pirates can play into the hands of police.
-"Technological determinants" don't have ANY human allegiances.
-They're not black or white, or Establishment or Underground,
-or pro-or-anti anything.
-
-Godwin complained at length about what he called "the Clever Hobbyist
-hypothesis" --the assumption that the "hacker" you're busting is clearly
-a technical genius, and must therefore by searched with extreme thoroughness.
-So: from the law's point of view, why risk missing anything? Take the works.
-Take the guy's computer. Take his books. Take his notebooks.
-Take the electronic drafts of his love letters. Take his Walkman.
-Take his wife's computer. Take his dad's computer. Take his kid
-sister's computer. Take his employer's computer. Take his compact disks--
-they MIGHT be CD-ROM disks, cunningly disguised as pop music.
-Take his laser printer--he might have hidden something vital in the
-printer's 5meg of memory. Take his software manuals and hardware
-documentation. Take his science-fiction novels and his simulation-
-gaming books. Take his Nintendo Game-Boy and his Pac-Man arcade game.
-Take his answering machine, take his telephone out of the wall.
-Take anything remotely suspicious.
-
-Godwin pointed out that most "hackers" are not, in fact, clever
-genius hobbyists. Quite a few are crooks and grifters who don't
-have much in the way of technical sophistication; just some rule-of-thumb
-rip-off techniques. The same goes for most fifteen-year-olds who've
-downloaded a code-scanning program from a pirate board. There's no
-real need to seize everything in sight. It doesn't require an entire
-computer system and ten thousand disks to prove a case in court.
-
-What if the computer is the instrumentality of a crime? someone demanded.
-
-Godwin admitted quietly that the doctrine of seizing the instrumentality
-of a crime was pretty well established in the American legal system.
-
-The meeting broke up. Godwin and Kapor had to leave. Kapor was testifying
-next morning before the Massachusetts Department Of Public Utility,
-about ISDN narrowband wide-area networking.
-
-As soon as they were gone, Thackeray seemed elated.
-She had taken a great risk with this. Her colleagues had not,
-in fact, torn Kapor and Godwin's heads off. She was very proud of them,
-and told them so.
-
-"Did you hear what Godwin said about INSTRUMENTALITY OF A CRIME?"
-she exulted, to nobody in particular. "Wow, that means
-MITCH ISN'T GOING TO SUE ME."
-
-#
-
-America's computer police are an interesting group.
-As a social phenomenon they are far more interesting,
-and far more important, than teenage phone phreaks
-and computer hackers. First, they're older and wiser;
-not dizzy hobbyists with leaky morals, but seasoned adult
-professionals with all the responsibilities of public service.
-And, unlike hackers, they possess not merely TECHNICAL
-power alone, but heavy-duty legal and social authority.
-
-And, very interestingly, they are just as much at
-sea in cyberspace as everyone else. They are not
-happy about this. Police are authoritarian by nature,
-and prefer to obey rules and precedents. (Even those police
-who secretly enjoy a fast ride in rough territory will soberly
-disclaim any "cowboy" attitude.) But in cyberspace there ARE
-no rules and precedents. They are groundbreaking pioneers,
-Cyberspace Rangers, whether they like it or not.
-
-In my opinion, any teenager enthralled by computers,
-fascinated by the ins and outs of computer security,
-and attracted by the lure of specialized forms of knowledge and power,
-would do well to forget all about "hacking" and set his (or her)
-sights on becoming a fed. Feds can trump hackers at almost every
-single thing hackers do, including gathering intelligence,
-undercover disguise, trashing, phone-tapping, building dossiers,
-networking, and infiltrating computer systems--CRIMINAL computer systems.
-Secret Service agents know more about phreaking, coding and carding
-than most phreaks can find out in years, and when it comes to viruses,
-break-ins, software bombs and trojan horses, Feds have direct access to red-hot
-confidential information that is only vague rumor in the underground.
-
-And if it's an impressive public rep you're after, there are few people
-in the world who can be so chillingly impressive as a well-trained,
-well-armed United States Secret Service agent.
-
-Of course, a few personal sacrifices are necessary in order to obtain
-that power and knowledge. First, you'll have the galling discipline
-of belonging to a large organization; but the world of computer crime
-is still so small, and so amazingly fast-moving, that it will remain
-spectacularly fluid for years to come. The second sacrifice is that
-you'll have to give up ripping people off. This is not a great loss.
-Abstaining from the use of illegal drugs, also necessary, will be a boon
-to your health.
-
-A career in computer security is not a bad choice for a young man
-or woman today. The field will almost certainly expand drastically
-in years to come. If you are a teenager today, by the time you
-become a professional, the pioneers you have read about in this book
-will be the grand old men and women of the field, swamped by their many
-disciples and successors. Of course, some of them, like William P. Wood
-of the 1865 Secret Service, may well be mangled in the whirring machinery
-of legal controversy; but by the time you enter the computer-crime field,
-it may have stabilized somewhat, while remaining entertainingly challenging.
-
-But you can't just have a badge. You have to win it. First, there's the
-federal law enforcement training. And it's hard--it's a challenge.
-A real challenge--not for wimps and rodents.
-
-Every Secret Service agent must complete gruelling courses at the
-Federal Law Enforcement Training Center. (In fact, Secret Service
-agents are periodically re-trained during their entire careers.)
-
-In order to get a glimpse of what this might be like,
-I myself travelled to FLETC.
-
-#
-
-The Federal Law Enforcement Training Center is a 1500-acre facility
-on Georgia's Atlantic coast. It's a milieu of marshgrass, seabirds,
-damp, clinging sea-breezes, palmettos, mosquitos, and bats.
-Until 1974, it was a Navy Air Base, and still features a working runway,
-and some WWII vintage blockhouses and officers' quarters.
-The Center has since benefitted by a forty-million-dollar retrofit,
-but there's still enough forest and swamp on the facility for the
-Border Patrol to put in tracking practice.
-
-As a town, "Glynco" scarcely exists. The nearest real town is Brunswick,
-a few miles down Highway 17, where I stayed at the aptly named Marshview
-Holiday Inn. I had Sunday dinner at a seafood restaurant called "Jinright's,"
-where I feasted on deep-fried alligator tail. This local favorite was
-a heaped basket of bite-sized chunks of white, tender, almost fluffy
-reptile meat, steaming in a peppered batter crust. Alligator makes
-a culinary experience that's hard to forget, especially when liberally
-basted with homemade cocktail sauce from a Jinright squeeze-bottle.
-
-The crowded clientele were tourists, fishermen, local black folks
-in their Sunday best, and white Georgian locals who all seemed
-to bear an uncanny resemblance to Georgia humorist Lewis Grizzard.
-
-The 2,400 students from 75 federal agencies who make up the FLETC
-population scarcely seem to make a dent in the low-key local scene.
-The students look like tourists, and the teachers seem to have taken
-on much of the relaxed air of the Deep South. My host was Mr. Carlton
-Fitzpatrick, the Program Coordinator of the Financial Fraud Institute.
-Carlton Fitzpatrick is a mustached, sinewy, well-tanned Alabama native
-somewhere near his late forties, with a fondness for chewing tobacco,
-powerful computers, and salty, down-home homilies. We'd met before,
-at FCIC in Arizona.
-
-The Financial Fraud Institute is one of the nine divisions at FLETC.
-Besides Financial Fraud, there's Driver & Marine, Firearms,
-and Physical Training. These are specialized pursuits.
-There are also five general training divisions: Basic Training,
-Operations, Enforcement Techniques, Legal Division, and Behavioral Science.
-
-Somewhere in this curriculum is everything necessary to turn green college
-graduates into federal agents. First they're given ID cards. Then they get
-the rather miserable-looking blue coveralls known as "smurf suits."
-The trainees are assigned a barracks and a cafeteria, and immediately
-set on FLETC's bone-grinding physical training routine. Besides the
-obligatory daily jogging--(the trainers run up danger flags beside
-the track when the humidity rises high enough to threaten heat stroke)--
-here's the Nautilus machines, the martial arts, the survival skills. . . .
-
-The eighteen federal agencies who maintain on-site academies at FLETC
-employ a wide variety of specialized law enforcement units, some of them
-rather arcane. There's Border Patrol, IRS Criminal Investigation Division,
-Park Service, Fish and Wildlife, Customs, Immigration, Secret Service and
-the Treasury's uniformed subdivisions. . . . If you're a federal cop
-and you don't work for the FBI, you train at FLETC. This includes people
-as apparently obscure as the agents of the Railroad Retirement Board
-Inspector General. Or the Tennessee Valley Authority Police,
-who are in fact federal police officers, and can and do arrest criminals
-on the federal property of the Tennessee Valley Authority.
-
-And then there are the computer-crime people. All sorts, all backgrounds.
-Mr. Fitzpatrick is not jealous of his specialized knowledge. Cops all over,
-in every branch of service, may feel a need to learn what he can teach.
-Backgrounds don't matter much. Fitzpatrick himself was originally a
-Border Patrol veteran, then became a Border Patrol instructor at FLETC.
-His Spanish is still fluent--but he found himself strangely fascinated
-when the first computers showed up at the Training Center. Fitzpatrick
-did have a background in electrical engineering, and though he never
-considered himself a computer hacker, he somehow found himself writing
-useful little programs for this new and promising gizmo.
-
-He began looking into the general subject of computers and crime,
-reading Donn Parker's books and articles, keeping an ear cocked
-for war stories, useful insights from the field, the up-and-coming
-people of the local computer-crime and high-technology units. . . .
-Soon he got a reputation around FLETC as the resident "computer expert,"
-and that reputation alone brought him more exposure, more experience--
-until one day he looked around, and sure enough he WAS a federal
-computer-crime expert.
-
-In fact, this unassuming, genial man may be THE federal computer-crime expert.
-There are plenty of very good computer people, and plenty of very good
-federal investigators, but the area where these worlds of expertise overlap
-is very slim. And Carlton Fitzpatrick has been right at the center of that
-since 1985, the first year of the Colluquy, a group which owes much to
-his influence.
-
-He seems quite at home in his modest, acoustic-tiled office,
-with its Ansel Adams-style Western photographic art, a gold-framed
-Senior Instructor Certificate, and a towering bookcase crammed with
-three-ring binders with ominous titles such as Datapro Reports on
-Information Security and CFCA Telecom Security '90.
-
-The phone rings every ten minutes; colleagues show up at the door
-to chat about new developments in locksmithing or to shake their heads
-over the latest dismal developments in the BCCI global banking scandal.
-
-Carlton Fitzpatrick is a fount of computer-crime war-stories,
-related in an acerbic drawl. He tells me the colorful tale
-of a hacker caught in California some years back. He'd been
-raiding systems, typing code without a detectable break,
-for twenty, twenty-four, thirty-six hours straight. Not just
-logged on--TYPING. Investigators were baffled. Nobody
-could do that. Didn't he have to go to the bathroom?
-Was it some kind of automatic keyboard-whacking device
-that could actually type code?
-
-A raid on the suspect's home revealed a situation of astonishing squalor.
-The hacker turned out to be a Pakistani computer-science student who had
-flunked out of a California university. He'd gone completely underground
-as an illegal electronic immigrant, and was selling stolen phone-service
-to stay alive. The place was not merely messy and dirty, but in a state
-of psychotic disorder. Powered by some weird mix of culture shock,
-computer addiction, and amphetamines, the suspect had in fact been sitting
-in front of his computer for a day and a half straight, with snacks and
-drugs at hand on the edge of his desk and a chamber-pot under his chair.
-
-Word about stuff like this gets around in the hacker-tracker community.
-
-Carlton Fitzpatrick takes me for a guided tour by car around the
-FLETC grounds. One of our first sights is the biggest indoor
-firing range in the world. There are federal trainees in there,
-Fitzpatrick assures me politely, blasting away with a wide variety
-of automatic weapons: Uzis, Glocks, AK-47s. . . . He's willing to
-take me inside. I tell him I'm sure that's really interesting,
-but I'd rather see his computers. Carlton Fitzpatrick seems quite
-surprised and pleased. I'm apparently the first journalist he's ever
-seen who has turned down the shooting gallery in favor of microchips.
-
-Our next stop is a favorite with touring Congressmen: the three-mile
-long FLETC driving range. Here trainees of the Driver & Marine Division
-are taught high-speed pursuit skills, setting and breaking road-blocks,
-diplomatic security driving for VIP limousines. . . . A favorite FLETC
-pastime is to strap a passing Senator into the passenger seat beside a
-Driver & Marine trainer, hit a hundred miles an hour, then take it right into
-"the skid-pan," a section of greased track where two tons of Detroit iron
-can whip and spin like a hockey puck.
-
-Cars don't fare well at FLETC. First they're rifled again and again
-for search practice. Then they do 25,000 miles of high-speed
-pursuit training; they get about seventy miles per set
-of steel-belted radials. Then it's off to the skid pan,
-where sometimes they roll and tumble headlong in the grease.
-When they're sufficiently grease-stained, dented, and creaky,
-they're sent to the roadblock unit, where they're battered without pity.
-And finally then they're sacrificed to the Bureau of Alcohol,
-Tobacco and Firearms, whose trainees learn the ins and outs
-of car-bomb work by blowing them into smoking wreckage.
-
-There's a railroad box-car on the FLETC grounds, and a large
-grounded boat, and a propless plane; all training-grounds for searches.
-The plane sits forlornly on a patch of weedy tarmac next to an eerie
-blockhouse known as the "ninja compound," where anti-terrorism specialists
-practice hostage rescues. As I gaze on this creepy paragon of modern
-low-intensity warfare, my nerves are jangled by a sudden staccato outburst
-of automatic weapons fire, somewhere in the woods to my right.
-"Nine-millimeter," Fitzpatrick judges calmly.
-
-Even the eldritch ninja compound pales somewhat compared
-to the truly surreal area known as "the raid-houses."
-This is a street lined on both sides with nondescript
-concrete-block houses with flat pebbled roofs.
-They were once officers' quarters. Now they are training grounds.
-The first one to our left, Fitzpatrick tells me, has been specially
-adapted for computer search-and-seizure practice. Inside it has been
-wired for video from top to bottom, with eighteen pan-and-tilt
-remotely controlled videocams mounted on walls and in corners.
-Every movement of the trainee agent is recorded live by teachers,
-for later taped analysis. Wasted movements, hesitations, possibly lethal
-tactical mistakes--all are gone over in detail.
-
-Perhaps the weirdest single aspect of this building is its front door,
-scarred and scuffed all along the bottom, from the repeated impact,
-day after day, of federal shoe-leather.
-
-Down at the far end of the row of raid-houses some people are practicing
-a murder. We drive by slowly as some very young and rather nervous-looking
-federal trainees interview a heavyset bald man on the raid-house lawn.
-Dealing with murder takes a lot of practice; first you have to learn
-to control your own instinctive disgust and panic, then you have to learn
-to control the reactions of a nerve-shredded crowd of civilians,
-some of whom may have just lost a loved one, some of whom may be murderers--
-quite possibly both at once.
-
-A dummy plays the corpse. The roles of the bereaved, the morbidly curious,
-and the homicidal are played, for pay, by local Georgians: waitresses,
-musicians, most anybody who needs to moonlight and can learn a script.
-These people, some of whom are FLETC regulars year after year,
-must surely have one of the strangest jobs in the world.
-
-Something about the scene: "normal" people in a weird situation,
-standing around talking in bright Georgia sunshine, unsuccessfully
-pretending that something dreadful has gone on, while a dummy lies
-inside on faked bloodstains. . . . While behind this weird masquerade,
-like a nested set of Russian dolls, are grim future realities of real death,
-real violence, real murders of real people, that these young agents
-will really investigate, many times during their careers. . . .
-Over and over. . . . Will those anticipated murders look like this,
-feel like this--not as "real" as these amateur actors are trying to
-make it seem, but both as "real," and as numbingly unreal, as watching
-fake people standing around on a fake lawn? Something about this scene
-unhinges me. It seems nightmarish to me, Kafkaesque. I simply don't
-know how to take it; my head is turned around; I don't know whether to laugh,
-cry, or just shudder.
-
-When the tour is over, Carlton Fitzpatrick and I talk about computers.
-For the first time cyberspace seems like quite a comfortable place.
-It seems very real to me suddenly, a place where I know what I'm talking about,
-a place I'm used to. It's real. "Real." Whatever.
-
-Carlton Fitzpatrick is the only person I've met in cyberspace circles
-who is happy with his present equipment. He's got a 5 Meg RAM PC with
-a 112 meg hard disk; a 660 meg's on the way. He's got a Compaq 386 desktop,
-and a Zenith 386 laptop with 120 meg. Down the hall is a NEC Multi-Sync 2A
-with a CD-ROM drive and a 9600 baud modem with four com-lines.
-There's a training minicomputer, and a 10-meg local mini just for the Center,
-and a lab-full of student PC clones and half-a-dozen Macs or so.
-There's a Data General MV 2500 with 8 meg on board and a 370 meg disk.
-
-Fitzpatrick plans to run a UNIX board on the Data General when he's
-finished beta-testing the software for it, which he wrote himself.
-It'll have E-mail features, massive files on all manner of computer-crime
-and investigation procedures, and will follow the computer-security
-specifics of the Department of Defense "Orange Book." He thinks
-it will be the biggest BBS in the federal government.
-
-Will it have Phrack on it? I ask wryly.
-
-Sure, he tells me. Phrack, TAP, Computer Underground Digest,
-all that stuff. With proper disclaimers, of course.
-
-I ask him if he plans to be the sysop. Running a system that size is very
-time-consuming, and Fitzpatrick teaches two three-hour courses every day.
-
-No, he says seriously, FLETC has to get its money worth out of the instructors.
-He thinks he can get a local volunteer to do it, a high-school student.
-
-He says a bit more, something I think about an Eagle Scout law-enforcement
-liaison program, but my mind has rocketed off in disbelief.
-
-"You're going to put a TEENAGER in charge of a federal security BBS?"
-I'm speechless. It hasn't escaped my notice that the FLETC Financial
-Fraud Institute is the ULTIMATE hacker-trashing target; there is stuff in here,
-stuff of such utter and consummate cool by every standard of the
-digital underground. . . .
-
-I imagine the hackers of my acquaintance, fainting dead-away from
-forbidden-knowledge greed-fits, at the mere prospect of cracking
-the superultra top-secret computers used to train the Secret Service
-in computer-crime. . . .
-
-"Uhm, Carlton," I babble, "I'm sure he's a really nice kid and all,
-but that's a terrible temptation to set in front of somebody who's,
-you know, into computers and just starting out. . . ."
-
-"Yeah," he says, "that did occur to me." For the first time I begin
-to suspect that he's pulling my leg.
-
-He seems proudest when he shows me an ongoing project called JICC,
-Joint Intelligence Control Council. It's based on the services provided
-by EPIC, the El Paso Intelligence Center, which supplies data and intelligence
-to the Drug Enforcement Administration, the Customs Service, the Coast Guard,
-and the state police of the four southern border states. Certain EPIC files
-can now be accessed by drug-enforcement police of Central America,
-South America and the Caribbean, who can also trade information
-among themselves. Using a telecom program called "White Hat,"
-written by two brothers named Lopez from the Dominican Republic,
-police can now network internationally on inexpensive PCs.
-Carlton Fitzpatrick is teaching a class of drug-war agents
-from the Third World, and he's very proud of their progress.
-Perhaps soon the sophisticated smuggling networks of the
-Medellin Cartel will be matched by a sophisticated computer
-network of the Medellin Cartel's sworn enemies. They'll track boats,
-track contraband, track the international drug-lords who now leap over
-borders with great ease, defeating the police through the clever use
-of fragmented national jurisdictions.
-
-JICC and EPIC must remain beyond the scope of this book.
-They seem to me to be very large topics fraught with complications
-that I am not fit to judge. I do know, however, that the international,
-computer-assisted networking of police, across national boundaries,
-is something that Carlton Fitzpatrick considers very important,
-a harbinger of a desirable future. I also know that networks
-by their nature ignore physical boundaries. And I also know
-that where you put communications you put a community,
-and that when those communities become self-aware
-they will fight to preserve themselves and to expand their influence.
-I make no judgements whether this is good or bad.
-It's just cyberspace; it's just the way things are.
-
-I asked Carlton Fitzpatrick what advice he would have for
-a twenty-year-old who wanted to shine someday in the world
-of electronic law enforcement.
-
-He told me that the number one rule was simply not to be
-scared of computers. You don't need to be an obsessive
-"computer weenie," but you mustn't be buffaloed just because
-some machine looks fancy. The advantages computers give
-smart crooks are matched by the advantages they give smart cops.
-Cops in the future will have to enforce the law "with their heads,
-not their holsters." Today you can make good cases without ever
-leaving your office. In the future, cops who resist the computer
-revolution will never get far beyond walking a beat.
-
-I asked Carlton Fitzpatrick if he had some single message for the public;
-some single thing that he would most like the American public to know
-about his work.
-
-He thought about it while. "Yes," he said finally. "TELL me the rules,
-and I'll TEACH those rules!" He looked me straight in the eye.
-"I do the best that I can."
-
-
-
-PART FOUR: THE CIVIL LIBERTARIANS
-
-
-The story of the Hacker Crackdown, as we have followed it thus far,
-has been technological, subcultural, criminal and legal.
-The story of the Civil Libertarians, though it partakes
-of all those other aspects, is profoundly and thoroughly POLITICAL.
-
-In 1990, the obscure, long-simmering struggle over the ownership
-and nature of cyberspace became loudly and irretrievably public.
-People from some of the oddest corners of American society suddenly
-found themselves public figures. Some of these people found this
-situation much more than they had ever bargained for. They backpedalled,
-and tried to retreat back to the mandarin obscurity of their cozy
-subcultural niches. This was generally to prove a mistake.
-
-But the civil libertarians seized the day in 1990. They found themselves
-organizing, propagandizing, podium-pounding, persuading, touring,
-negotiating, posing for publicity photos, submitting to interviews,
-squinting in the limelight as they tried a tentative, but growingly
-sophisticated, buck-and-wing upon the public stage.
-
-It's not hard to see why the civil libertarians should have
-this competitive advantage.
-
-The hackers of the digital underground are an hermetic elite.
-They find it hard to make any remotely convincing case for
-their actions in front of the general public. Actually,
-hackers roundly despise the "ignorant" public, and have never
-trusted the judgement of "the system." Hackers do propagandize,
-but only among themselves, mostly in giddy, badly spelled manifestos
-of class warfare, youth rebellion or naive techie utopianism.
-Hackers must strut and boast in order to establish and preserve
-their underground reputations. But if they speak out too loudly
-and publicly, they will break the fragile surface-tension of the underground,
-and they will be harrassed or arrested. Over the longer term,
-most hackers stumble, get busted, get betrayed, or simply give up.
-As a political force, the digital underground is hamstrung.
-
-The telcos, for their part, are an ivory tower under protracted seige.
-They have plenty of money with which to push their calculated public image,
-but they waste much energy and goodwill attacking one another with
-slanderous and demeaning ad campaigns. The telcos have suffered
-at the hands of politicians, and, like hackers, they don't trust
-the public's judgement. And this distrust may be well-founded.
-Should the general public of the high-tech 1990s come to understand
-its own best interests in telecommunications, that might well pose
-a grave threat to the specialized technical power and authority
-that the telcos have relished for over a century. The telcos do
-have strong advantages: loyal employees, specialized expertise,
-influence in the halls of power, tactical allies in law enforcement,
-and unbelievably vast amounts of money. But politically speaking, they lack
-genuine grassroots support; they simply don't seem to have many friends.
-
-Cops know a lot of things other people don't know.
-But cops willingly reveal only those aspects of their
-knowledge that they feel will meet their institutional
-purposes and further public order. Cops have respect,
-they have responsibilities, they have power in the streets
-and even power in the home, but cops don't do particularly
-well in limelight. When pressed, they will step out in the
-public gaze to threaten bad-guys, or to cajole prominent citizens,
-or perhaps to sternly lecture the naive and misguided.
-But then they go back within their time-honored fortress
-of the station-house, the courtroom and the rule-book.
-
-The electronic civil libertarians, however, have proven to be
-born political animals. They seemed to grasp very early on
-the postmodern truism that communication is power. Publicity is power.
-Soundbites are power. The ability to shove one's issue onto the public
-agenda--and KEEP IT THERE--is power. Fame is power. Simple personal
-fluency and eloquence can be power, if you can somehow catch the
-public's eye and ear.
-
-The civil libertarians had no monopoly on "technical power"--
-though they all owned computers, most were not particularly
-advanced computer experts. They had a good deal of money,
-but nowhere near the earthshaking wealth and the galaxy
-of resources possessed by telcos or federal agencies.
-They had no ability to arrest people. They carried
-out no phreak and hacker covert dirty-tricks.
-
-But they really knew how to network.
-
-Unlike the other groups in this book, the civil libertarians
-have operated very much in the open, more or less right
-in the public hurly-burly. They have lectured audiences galore
-and talked to countless journalists, and have learned to
-refine their spiels. They've kept the cameras clicking,
-kept those faxes humming, swapped that email,
-run those photocopiers on overtime, licked envelopes
-and spent small fortunes on airfare and long-distance.
-In an information society, this open, overt, obvious activity
-has proven to be a profound advantage.
-
-In 1990, the civil libertarians of cyberspace assembled
-out of nowhere in particular, at warp speed. This "group"
-(actually, a networking gaggle of interested parties
-which scarcely deserves even that loose term) has almost nothing
-in the way of formal organization. Those formal civil libertarian
-organizations which did take an interest in cyberspace issues,
-mainly the Computer Professionals for Social Responsibility
-and the American Civil Liberties Union, were carried along
-by events in 1990, and acted mostly as adjuncts,
-underwriters or launching-pads.
-
-The civil libertarians nevertheless enjoyed the greatest success
-of any of the groups in the Crackdown of 1990. At this writing,
-their future looks rosy and the political initiative is firmly in their hands.
-This should be kept in mind as we study the highly unlikely lives
-and lifestyles of the people who actually made this happen.
-
-#
-
-In June 1989, Apple Computer, Inc., of Cupertino,
-California, had a problem. Someone had illicitly copied
-a small piece of Apple's proprietary software, software
-which controlled an internal chip driving the Macintosh
-screen display. This Color QuickDraw source code was
-a closely guarded piece of Apple's intellectual property.
-Only trusted Apple insiders were supposed to possess it.
-
-But the "NuPrometheus League" wanted things otherwise.
-This person (or persons) made several illicit copies
-of this source code, perhaps as many as two dozen.
-He (or she, or they) then put those illicit floppy disks
-into envelopes and mailed them to people all over America:
-people in the computer industry who were associated with,
-but not directly employed by, Apple Computer.
-
-The NuPrometheus caper was a complex, highly ideological,
-and very hacker-like crime. Prometheus, it will be recalled,
-stole the fire of the Gods and gave this potent gift to the
-general ranks of downtrodden mankind. A similar god-in-the-manger
-attitude was implied for the corporate elite of Apple Computer,
-while the "Nu" Prometheus had himself cast in the role of rebel demigod.
-The illicitly copied data was given away for free.
-
-The new Prometheus, whoever he was, escaped the
-fate of the ancient Greek Prometheus, who was chained
-to a rock for centuries by the vengeful gods while an eagle
-tore and ate his liver. On the other hand, NuPrometheus
-chickened out somewhat by comparison with his role model.
-The small chunk of Color QuickDraw code he had filched
-and replicated was more or less useless to Apple's
-industrial rivals (or, in fact, to anyone else).
-Instead of giving fire to mankind, it was more as if
-NuPrometheus had photocopied the schematics for part of a Bic lighter.
-The act was not a genuine work of industrial espionage.
-It was best interpreted as a symbolic, deliberate slap
-in the face for the Apple corporate heirarchy.
-
-Apple's internal struggles were well-known in the industry. Apple's founders,
-Jobs and Wozniak, had both taken their leave long since. Their raucous core
-of senior employees had been a barnstorming crew of 1960s Californians,
-many of them markedly less than happy with the new button-down multimillion
-dollar regime at Apple. Many of the programmers and developers who had
-invented the Macintosh model in the early 1980s had also taken their leave of
-the company. It was they, not the current masters of Apple's corporate fate,
-who had invented the stolen Color QuickDraw code. The NuPrometheus stunt
-was well-calculated to wound company morale.
-
-Apple called the FBI. The Bureau takes an interest in high-profile
-intellectual-property theft cases, industrial espionage and theft
-of trade secrets. These were likely the right people to call,
-and rumor has it that the entities responsible were in fact discovered
-by the FBI, and then quietly squelched by Apple management. NuPrometheus
-was never publicly charged with a crime, or prosecuted, or jailed.
-But there were no further illicit releases of Macintosh internal software.
-Eventually the painful issue of NuPrometheus was allowed to fade.
-
-In the meantime, however, a large number of puzzled bystanders
-found themselves entertaining surprise guests from the FBI.
-
-One of these people was John Perry Barlow. Barlow is a most unusual man,
-difficult to describe in conventional terms. He is perhaps best known as
-a songwriter for the Grateful Dead, for he composed lyrics for
-"Hell in a Bucket," "Picasso Moon," "Mexicali Blues," "I Need a Miracle,"
-and many more; he has been writing for the band since 1970.
-
-Before we tackle the vexing question as to why a rock lyricist
-should be interviewed by the FBI in a computer-crime case,
-it might be well to say a word or two about the Grateful Dead.
-The Grateful Dead are perhaps the most successful and long-lasting
-of the numerous cultural emanations from the Haight-Ashbury district
-of San Francisco, in the glory days of Movement politics and
-lysergic transcendance. The Grateful Dead are a nexus, a veritable
-whirlwind, of applique decals, psychedelic vans, tie-dyed T-shirts,
-earth-color denim, frenzied dancing and open and unashamed drug use.
-The symbols, and the realities, of Californian freak power surround
-the Grateful Dead like knotted macrame.
-
-The Grateful Dead and their thousands of Deadhead devotees
-are radical Bohemians. This much is widely understood.
-Exactly what this implies in the 1990s is rather more problematic.
-
-The Grateful Dead are among the world's most popular
-and wealthy entertainers: number 20, according to Forbes magazine,
-right between M.C. Hammer and Sean Connery. In 1990, this jeans-clad
-group of purported raffish outcasts earned seventeen million dollars.
-They have been earning sums much along this line for quite some time now.
-
-And while the Dead are not investment bankers or three-piece-suit
-tax specialists--they are, in point of fact, hippie musicians--
-this money has not been squandered in senseless Bohemian excess.
-The Dead have been quietly active for many years, funding various
-worthy activities in their extensive and widespread cultural community.
-
-The Grateful Dead are not conventional players in the American
-power establishment. They nevertheless are something of a force
-to be reckoned with. They have a lot of money and a lot of friends
-in many places, both likely and unlikely.
-
-The Dead may be known for back-to-the-earth environmentalist rhetoric,
-but this hardly makes them anti-technological Luddites. On the contrary,
-like most rock musicians, the Grateful Dead have spent their entire adult
-lives in the company of complex electronic equipment. They have funds to burn
-on any sophisticated tool and toy that might happen to catch their fancy.
-And their fancy is quite extensive.
-
-The Deadhead community boasts any number of recording engineers,
-lighting experts, rock video mavens, electronic technicians
-of all descriptions. And the drift goes both ways. Steve Wozniak,
-Apple's co-founder, used to throw rock festivals. Silicon Valley rocks out.
-
-These are the 1990s, not the 1960s. Today, for a surprising number of people
-all over America, the supposed dividing line between Bohemian and technician
-simply no longer exists. People of this sort may have a set of windchimes
-and a dog with a knotted kerchief 'round its neck, but they're also quite
-likely to own a multimegabyte Macintosh running MIDI synthesizer software
-and trippy fractal simulations. These days, even Timothy Leary himself,
-prophet of LSD, does virtual-reality computer-graphics demos in
-his lecture tours.
-
-John Perry Barlow is not a member of the Grateful Dead. He is, however,
-a ranking Deadhead.
-
-Barlow describes himself as a "techno-crank." A vague term like
-"social activist" might not be far from the mark, either.
-But Barlow might be better described as a "poet"--if one keeps in mind
-Percy Shelley's archaic definition of poets as "unacknowledged legislators
-of the world."
-
-Barlow once made a stab at acknowledged legislator status. In 1987,
-he narrowly missed the Republican nomination for a seat in the
-Wyoming State Senate. Barlow is a Wyoming native, the third-generation
-scion of a well-to-do cattle-ranching family. He is in his early forties,
-married and the father of three daughters.
-
-Barlow is not much troubled by other people's narrow notions of consistency.
-In the late 1980s, this Republican rock lyricist cattle rancher sold his ranch
-and became a computer telecommunications devotee.
-
-The free-spirited Barlow made this transition with ease. He genuinely
-enjoyed computers. With a beep of his modem, he leapt from small-town
-Pinedale, Wyoming, into electronic contact with a large and lively crowd
-of bright, inventive, technological sophisticates from all over the world.
-Barlow found the social milieu of computing attractive: its fast-lane pace,
-its blue-sky rhetoric, its open-endedness. Barlow began dabbling in
-computer journalism, with marked success, as he was a quick study,
-and both shrewd and eloquent. He frequently travelled to San Francisco
-to network with Deadhead friends. There Barlow made extensive contacts
-throughout the Californian computer community, including friendships
-among the wilder spirits at Apple.
-
-In May 1990, Barlow received a visit from a local Wyoming agent of the FBI.
-The NuPrometheus case had reached Wyoming.
-
-Barlow was troubled to find himself under investigation in an
-area of his interests once quite free of federal attention.
-He had to struggle to explain the very nature of computer-crime
-to a headscratching local FBI man who specialized in cattle-rustling.
-Barlow, chatting helpfully and demonstrating the wonders of his modem
-to the puzzled fed, was alarmed to find all "hackers" generally under
-FBI suspicion as an evil influence in the electronic community.
-The FBI, in pursuit of a hacker called "NuPrometheus," were tracing
-attendees of a suspect group called the Hackers Conference.
-
-The Hackers Conference, which had been started in 1984, was a
-yearly Californian meeting of digital pioneers and enthusiasts.
-The hackers of the Hackers Conference had little if anything to do
-with the hackers of the digital underground. On the contrary,
-the hackers of this conference were mostly well-to-do Californian
-high-tech CEOs, consultants, journalists and entrepreneurs.
-(This group of hackers were the exact sort of "hackers"
-most likely to react with militant fury at any criminal
-degradation of the term "hacker.")
-
-Barlow, though he was not arrested or accused of a crime,
-and though his computer had certainly not gone out the door,
-was very troubled by this anomaly. He carried the word to the Well.
-
-Like the Hackers Conference, "the Well" was an emanation of the
-Point Foundation. Point Foundation, the inspiration of a wealthy
-Californian 60s radical named Stewart Brand, was to be a major
-launch-pad of the civil libertarian effort.
-
-Point Foundation's cultural efforts, like those of their fellow Bay Area
-Californians the Grateful Dead, were multifaceted and multitudinous.
-Rigid ideological consistency had never been a strong suit of the
-Whole Earth Catalog. This Point publication had enjoyed a strong
-vogue during the late 60s and early 70s, when it offered hundreds
-of practical (and not so practical) tips on communitarian living,
-environmentalism, and getting back-to-the-land. The Whole Earth Catalog,
-and its sequels, sold two and half million copies and won a
-National Book Award.
-
-With the slow collapse of American radical dissent, the Whole Earth Catalog
-had slipped to a more modest corner of the cultural radar; but in its
-magazine incarnation, CoEvolution Quarterly, the Point Foundation
-continued to offer a magpie potpourri of "access to tools and ideas."
-
-CoEvolution Quarterly, which started in 1974, was never a widely
-popular magazine. Despite periodic outbreaks of millenarian fervor,
-CoEvolution Quarterly failed to revolutionize Western civilization
-and replace leaden centuries of history with bright new Californian paradigms.
-Instead, this propaganda arm of Point Foundation cakewalked a fine line between
-impressive brilliance and New Age flakiness. CoEvolution Quarterly carried
-no advertising, cost a lot, and came out on cheap newsprint with modest
-black-and-white graphics. It was poorly distributed, and spread mostly
-by subscription and word of mouth.
-
-It could not seem to grow beyond 30,000 subscribers.
-And yet--it never seemed to shrink much, either.
-Year in, year out, decade in, decade out, some strange
-demographic minority accreted to support the magazine.
-The enthusiastic readership did not seem to have much
-in the way of coherent politics or ideals. It was sometimes
-hard to understand what held them together (if the often bitter
-debate in the letter-columns could be described as "togetherness").
-
-But if the magazine did not flourish, it was resilient; it got by.
-Then, in 1984, the birth-year of the Macintosh computer,
-CoEvolution Quarterly suddenly hit the rapids. Point Foundation
-had discovered the computer revolution. Out came the Whole Earth
-Software Catalog of 1984, arousing headscratching doubts among
-the tie-dyed faithful, and rabid enthusiasm among the nascent
-"cyberpunk" milieu, present company included. Point Foundation
-started its yearly Hackers Conference, and began to take an
-extensive interest in the strange new possibilities of
-digital counterculture. CoEvolution Quarterlyfolded its teepee,
-replaced by Whole Earth Software Review and eventually by Whole Earth
-Review (the magazine's present incarnation, currently under
-the editorship of virtual-reality maven Howard Rheingold).
-
-1985 saw the birth of the "WELL"--the "Whole Earth 'Lectronic Link."
-The Well was Point Foundation's bulletin board system.
-
-As boards went, the Well was an anomaly from the beginning,
-and remained one. It was local to San Francisco.
-It was huge, with multiple phonelines and enormous files
-of commentary. Its complex UNIX-based software might be
-most charitably described as "user-opaque." It was run on
-a mainframe out of the rambling offices of a non-profit
-cultural foundation in Sausalito. And it was crammed with
-fans of the Grateful Dead.
-
-Though the Well was peopled by chattering hipsters of the Bay Area
-counterculture, it was by no means a "digital underground" board.
-Teenagers were fairly scarce; most Well users (known as "Wellbeings")
-were thirty- and forty-something Baby Boomers. They tended to work
-in the information industry: hardware, software, telecommunications,
-media, entertainment. Librarians, academics, and journalists were
-especially common on the Well, attracted by Point Foundation's
-open-handed distribution of "tools and ideas."
-
-There were no anarchy files on the Well, scarcely a
-dropped hint about access codes or credit-card theft.
-No one used handles. Vicious "flame-wars" were held to
-a comparatively civilized rumble. Debates were sometimes sharp,
-but no Wellbeing ever claimed that a rival had disconnected his phone,
-trashed his house, or posted his credit card numbers.
-
-The Well grew slowly as the 1980s advanced. It charged a modest sum
-for access and storage, and lost money for years--but not enough to hamper
-the Point Foundation, which was nonprofit anyway. By 1990, the Well
-had about five thousand users. These users wandered about a gigantic
-cyberspace smorgasbord of "Conferences", each conference itself consisting
-of a welter of "topics," each topic containing dozens, sometimes hundreds
-of comments, in a tumbling, multiperson debate that could last for months
-or years on end.
-
-
-In 1991, the Well's list of conferences looked like this:
-
-
-CONFERENCES ON THE WELL
-
-WELL "Screenzine" Digest (g zine)
-
-Best of the WELL - vintage material - (g best)
-
-Index listing of new topics in all conferences - (g newtops)
-
-Business - Education
-----------------------
-
-Apple Library Users Group(g alug) Agriculture (g agri)
-Brainstorming (g brain) Classifieds (g cla)
-Computer Journalism (g cj) Consultants (g consult)
-Consumers (g cons) Design (g design)
-Desktop Publishing (g desk) Disability (g disability)
-Education (g ed) Energy (g energy91)
-Entrepreneurs (g entre) Homeowners (g home)
-Indexing (g indexing) Investments (g invest)
-Kids91 (g kids) Legal (g legal)
-One Person Business (g one)
-Periodical/newsletter (g per)
-Telecomm Law (g tcl) The Future (g fut)
-Translators (g trans) Travel (g tra)
-Work (g work)
-
-Electronic Frontier Foundation (g eff)
-Computers, Freedom & Privacy (g cfp)
-Computer Professionals for Social Responsibility (g cpsr)
-
-Social - Political - Humanities
----------------------------------
-
-Aging (g gray) AIDS (g aids)
-Amnesty International (g amnesty) Archives (g arc)
-Berkeley (g berk) Buddhist (g wonderland)
-Christian (g cross) Couples (g couples)
-Current Events (g curr) Dreams (g dream)
-Drugs (g dru) East Coast (g east)
-Emotional Health@@@@ (g private) Erotica (g eros)
-Environment (g env) Firearms (g firearms)
-First Amendment (g first) Fringes of Reason (g fringes)
-Gay (g gay) Gay (Private)# (g gaypriv)
-Geography (g geo) German (g german)
-Gulf War (g gulf) Hawaii (g aloha)
-Health (g heal) History (g hist)
-Holistic (g holi) Interview (g inter)
-Italian (g ital) Jewish (g jew)
-Liberty (g liberty) Mind (g mind)
-Miscellaneous (g misc) Men on the WELL@@ (g mow)
-Network Integration (g origin) Nonprofits (g non)
-North Bay (g north) Northwest (g nw)
-Pacific Rim (g pacrim) Parenting (g par)
-Peace (g pea) Peninsula (g pen)
-Poetry (g poetry) Philosophy (g phi)
-Politics (g pol) Psychology (g psy)
-Psychotherapy (g therapy) Recovery## (g recovery)
-San Francisco (g sanfran) Scams (g scam)
-Sexuality (g sex) Singles (g singles)
-Southern (g south) Spanish (g spanish)
-Spirituality (g spirit) Tibet (g tibet)
-Transportation (g transport) True Confessions (g tru)
-Unclear (g unclear) WELL Writer's Workshop@@@(g www)
-Whole Earth (g we) Women on the WELL@(g wow)
-Words (g words) Writers (g wri)
-
-@@@@Private Conference - mail wooly for entry
-@@@Private conference - mail sonia for entry
-@@Private conference - mail flash for entry
-@ Private conference - mail reva for entry
-# Private Conference - mail hudu for entry
-## Private Conference - mail dhawk for entry
-
-Arts - Recreation - Entertainment
------------------------------------
-ArtCom Electronic Net (g acen)
-Audio-Videophilia (g aud)
-Bicycles (g bike) Bay Area Tonight@@(g bat)
-Boating (g wet) Books (g books)
-CD's (g cd) Comics (g comics)
-Cooking (g cook) Flying (g flying)
-Fun (g fun) Games (g games)
-Gardening (g gard) Kids (g kids)
-Nightowls@ (g owl) Jokes (g jokes)
-MIDI (g midi) Movies (g movies)
-Motorcycling (g ride) Motoring (g car)
-Music (g mus) On Stage (g onstage)
-Pets (g pets) Radio (g rad)
-Restaurant (g rest) Science Fiction (g sf)
-Sports (g spo) Star Trek (g trek)
-Television (g tv) Theater (g theater)
-Weird (g weird) Zines/Factsheet Five(g f5)
-@Open from midnight to 6am
-@@Updated daily
-
-Grateful Dead
--------------
-Grateful Dead (g gd) Deadplan@ (g dp)
-Deadlit (g deadlit) Feedback (g feedback)
-GD Hour (g gdh) Tapes (g tapes)
-Tickets (g tix) Tours (g tours)
-
-@Private conference - mail tnf for entry
-
-Computers
------------
-AI/Forth/Realtime (g realtime) Amiga (g amiga)
-Apple (g app) Computer Books (g cbook)
-Art & Graphics (g gra) Hacking (g hack)
-HyperCard (g hype) IBM PC (g ibm)
-LANs (g lan) Laptop (g lap)
-Macintosh (g mac) Mactech (g mactech)
-Microtimes (g microx) Muchomedia (g mucho)
-NeXt (g next) OS/2 (g os2)
-Printers (g print) Programmer's Net (g net)
-Siggraph (g siggraph) Software Design (g sdc)
-Software/Programming (g software)
-Software Support (g ssc)
-Unix (g unix) Windows (g windows)
-Word Processing (g word)
-
-Technical - Communications
-----------------------------
-Bioinfo (g bioinfo) Info (g boing)
-Media (g media) NAPLPS (g naplps)
-Netweaver (g netweaver) Networld (g networld)
-Packet Radio (g packet) Photography (g pho)
-Radio (g rad) Science (g science)
-Technical Writers (g tec) Telecommunications(g tele)
-Usenet (g usenet) Video (g vid)
-Virtual Reality (g vr)
-
-The WELL Itself
----------------
-Deeper (g deeper) Entry (g ent)
-General (g gentech) Help (g help)
-Hosts (g hosts) Policy (g policy)
-System News (g news) Test (g test)
-
-The list itself is dazzling, bringing to the untutored eye
-a dizzying impression of a bizarre milieu of mountain-climbing
-Hawaiian holistic photographers trading true-life confessions
-with bisexual word-processing Tibetans.
-
-But this confusion is more apparent than real. Each of these conferences
-was a little cyberspace world in itself, comprising dozens and perhaps
-hundreds of sub-topics. Each conference was commonly frequented by
-a fairly small, fairly like-minded community of perhaps a few dozen people.
-It was humanly impossible to encompass the entire Well (especially since
-access to the Well's mainframe computer was billed by the hour).
-Most long-time users contented themselves with a few favorite
-topical neighborhoods, with the occasional foray elsewhere
-for a taste of exotica. But especially important news items,
-and hot topical debates, could catch the attention of the entire
-Well community.
-
-Like any community, the Well had its celebrities, and John Perry Barlow,
-the silver-tongued and silver-modemed lyricist of the Grateful Dead,
-ranked prominently among them. It was here on the Well that Barlow
-posted his true-life tale of computer-crime encounter with the FBI.
-
-The story, as might be expected, created a great stir. The Well was
-already primed for hacker controversy. In December 1989, Harper's magazine
-had hosted a debate on the Well about the ethics of illicit computer intrusion.
-While over forty various computer-mavens took part, Barlow proved a star
-in the debate. So did "Acid Phreak" and "Phiber Optik," a pair of young
-New York hacker-phreaks whose skills at telco switching-station intrusion
-were matched only by their apparently limitless hunger for fame.
-The advent of these two boldly swaggering outlaws in the precincts
-of the Well created a sensation akin to that of Black Panthers
-at a cocktail party for the radically chic.
-
-Phiber Optik in particular was to seize the day in 1990.
-A devotee of the 2600 circle and stalwart of the New York
-hackers' group "Masters of Deception," Phiber Optik was
-a splendid exemplar of the computer intruder as committed dissident.
-The eighteen-year-old Optik, a high-school dropout and part-time
-computer repairman, was young, smart, and ruthlessly obsessive,
-a sharp-dressing, sharp-talking digital dude who was utterly
-and airily contemptuous of anyone's rules but his own.
-By late 1991, Phiber Optik had appeared in Harper's,
-Esquire, The New York Times, in countless public debates
-and conventions, even on a television show hosted by Geraldo Rivera.
-
-Treated with gingerly respect by Barlow and other Well mavens,
-Phiber Optik swiftly became a Well celebrity. Strangely, despite
-his thorny attitude and utter single-mindedness, Phiber Optik seemed
-to arouse strong protective instincts in most of the people who met him.
-He was great copy for journalists, always fearlessly ready to swagger,
-and, better yet, to actually DEMONSTRATE some off-the-wall digital stunt.
-He was a born media darling.
-
-Even cops seemed to recognize that there was something peculiarly unworldly
-and uncriminal about this particular troublemaker. He was so bold,
-so flagrant, so young, and so obviously doomed, that even those
-who strongly disapproved of his actions grew anxious for his welfare,
-and began to flutter about him as if he were an endangered seal pup.
-
-In January 24, 1990 (nine days after the Martin Luther King Day Crash),
-Phiber Optik, Acid Phreak, and a third NYC scofflaw named Scorpion were
-raided by the Secret Service. Their computers went out the door,
-along with the usual blizzard of papers, notebooks, compact disks,
-answering machines, Sony Walkmans, etc. Both Acid Phreak and
-Phiber Optik were accused of having caused the Crash.
-
-The mills of justice ground slowly. The case eventually fell into
-the hands of the New York State Police. Phiber had lost his machinery
-in the raid, but there were no charges filed against him for over a year.
-His predicament was extensively publicized on the Well, where it caused
-much resentment for police tactics. It's one thing to merely hear about
-a hacker raided or busted; it's another to see the police attacking someone
-you've come to know personally, and who has explained his motives at length.
-Through the Harper's debate on the Well, it had become clear to the
-Wellbeings that Phiber Optik was not in fact going to "hurt anything."
-In their own salad days, many Wellbeings had tasted tear-gas in pitched
-street-battles with police. They were inclined to indulgence for
-acts of civil disobedience.
-
-Wellbeings were also startled to learn of the draconian thoroughness
-of a typical hacker search-and-seizure. It took no great stretch of
-imagination for them to envision themselves suffering much the same treatment.
-
-As early as January 1990, sentiment on the Well had already begun to sour,
-and people had begun to grumble that "hackers" were getting a raw deal
-from the ham-handed powers-that-be. The resultant issue of Harper's
-magazine posed the question as to whether computer-intrusion was a "crime"
-at all. As Barlow put it later: "I've begun to wonder if we wouldn't
-also regard spelunkers as desperate criminals if AT&T owned all the caves."
-
-In February 1991, more than a year after the raid on his home,
-Phiber Optik was finally arrested, and was charged with first-degree
-Computer Tampering and Computer Trespass, New York state offenses.
-He was also charged with a theft-of-service misdemeanor, involving a complex
-free-call scam to a 900 number. Phiber Optik pled guilty to the misdemeanor
-charge, and was sentenced to 35 hours of community service.
-
-This passing harassment from the unfathomable world of straight people
-seemed to bother Optik himself little if at all. Deprived of his computer
-by the January search-and-seizure, he simply bought himself a portable
-computer so the cops could no longer monitor the phone where he lived
-with his Mom, and he went right on with his depredations, sometimes on
-live radio or in front of television cameras.
-
-The crackdown raid may have done little to dissuade Phiber Optik,
-but its galling affect on the Wellbeings was profound. As 1990 rolled on,
-the slings and arrows mounted: the Knight Lightning raid,
-the Steve Jackson raid, the nation-spanning Operation Sundevil.
-The rhetoric of law enforcement made it clear that there was,
-in fact, a concerted crackdown on hackers in progress.
-
-The hackers of the Hackers Conference, the Wellbeings, and their ilk,
-did not really mind the occasional public misapprehension of "hacking;"
-if anything, this membrane of differentiation from straight society
-made the "computer community" feel different, smarter, better.
-They had never before been confronted, however, by a concerted
-vilification campaign.
-
-Barlow's central role in the counter-struggle was one of the major
-anomalies of 1990. Journalists investigating the controversy
-often stumbled over the truth about Barlow, but they commonly
-dusted themselves off and hurried on as if nothing had happened.
-It was as if it were TOO MUCH TO BELIEVE that a 1960s freak
-from the Grateful Dead had taken on a federal law enforcement operation
-head-to-head and ACTUALLY SEEMED TO BE WINNING!
-
-Barlow had no easily detectable power-base for a political struggle
-of this kind. He had no formal legal or technical credentials.
-Barlow was, however, a computer networker of truly stellar brilliance.
-He had a poet's gift of concise, colorful phrasing. He also had a
-journalist's shrewdness, an off-the-wall, self-deprecating wit,
-and a phenomenal wealth of simple personal charm.
-
-The kind of influence Barlow possessed is fairly common currency
-in literary, artistic, or musical circles. A gifted critic can
-wield great artistic influence simply through defining
-the temper of the times, by coining the catch-phrases
-and the terms of debate that become the common currency of the period.
-(And as it happened, Barlow WAS a part-time art critic,
-with a special fondness for the Western art of Frederic Remington.)
-
-Barlow was the first commentator to adopt William Gibson's
-striking science-fictional term "cyberspace" as a synonym
-for the present-day nexus of computer and telecommunications networks.
-Barlow was insistent that cyberspace should be regarded as
-a qualitatively new world, a "frontier." According to Barlow,
-the world of electronic communications, now made visible through
-the computer screen, could no longer be usefully regarded
-as just a tangle of high-tech wiring. Instead, it had become
-a PLACE, cyberspace, which demanded a new set of metaphors,
-a new set of rules and behaviors. The term, as Barlow employed it,
-struck a useful chord, and this concept of cyberspace was picked up
-by Time, Scientific American, computer police, hackers, and even
-Constitutional scholars. "Cyberspace" now seems likely to become
-a permanent fixture of the language.
-
-Barlow was very striking in person: a tall, craggy-faced, bearded,
-deep-voiced Wyomingan in a dashing Western ensemble of jeans, jacket,
-cowboy boots, a knotted throat-kerchief and an ever-present Grateful Dead
-cloisonne lapel pin.
-
-Armed with a modem, however, Barlow was truly in his element.
-Formal hierarchies were not Barlow's strong suit; he rarely missed
-a chance to belittle the "large organizations and their drones,"
-with their uptight, institutional mindset. Barlow was very much
-of the free-spirit persuasion, deeply unimpressed by brass-hats
-and jacks-in-office. But when it came to the digital grapevine,
-Barlow was a cyberspace ad-hocrat par excellence.
-
-There was not a mighty army of Barlows. There was only one Barlow,
-and he was a fairly anomolous individual. However, the situation only
-seemed to REQUIRE a single Barlow. In fact, after 1990, many people
-must have concluded that a single Barlow was far more than
-they'd ever bargained for.
-
-Barlow's querulous mini-essay about his encounter with the FBI
-struck a strong chord on the Well. A number of other free spirits
-on the fringes of Apple Computing had come under suspicion,
-and they liked it not one whit better than he did.
-
-One of these was Mitchell Kapor, the co-inventor of the spreadsheet
-program "Lotus 1-2-3" and the founder of Lotus Development Corporation.
-Kapor had written-off the passing indignity of being fingerprinted
-down at his own local Boston FBI headquarters, but Barlow's post
-made the full national scope of the FBI's dragnet clear to Kapor.
-The issue now had Kapor's full attention. As the Secret Service
-swung into anti-hacker operation nationwide in 1990, Kapor watched
-every move with deep skepticism and growing alarm.
-
-As it happened, Kapor had already met Barlow, who had interviewed Kapor
-for a California computer journal. Like most people who met Barlow,
-Kapor had been very taken with him. Now Kapor took it upon himself
-to drop in on Barlow for a heart-to-heart talk about the situation.
-
-Kapor was a regular on the Well. Kapor had been a devotee of the
-Whole Earth Catalogsince the beginning, and treasured a complete run
-of the magazine. And Kapor not only had a modem, but a private jet.
-In pursuit of the scattered high-tech investments of Kapor Enterprises Inc.,
-his personal, multi-million dollar holding company, Kapor commonly crossed
-state lines with about as much thought as one might give to faxing a letter.
-
-The Kapor-Barlow council of June 1990, in Pinedale, Wyoming, was the start
-of the Electronic Frontier Foundation. Barlow swiftly wrote a manifesto,
-"Crime and Puzzlement," which announced his, and Kapor's, intention
-to form a political organization to "raise and disburse funds for education,
-lobbying, and litigation in the areas relating to digital speech and the
-extension of the Constitution into Cyberspace."
-
-Furthermore, proclaimed the manifesto, the foundation would
-"fund, conduct, and support legal efforts to demonstrate
-that the Secret Service has exercised prior restraint on publications,
-limited free speech, conducted improper seizure of equipment and data,
-used undue force, and generally conducted itself in a fashion which
-is arbitrary, oppressive, and unconstitutional."
-
-"Crime and Puzzlement" was distributed far and wide through computer
-networking channels, and also printed in the Whole Earth Review.
-The sudden declaration of a coherent, politicized counter-strike
-from the ranks of hackerdom electrified the community. Steve Wozniak
-(perhaps a bit stung by the NuPrometheus scandal) swiftly offered
-to match any funds Kapor offered the Foundation.
-
-John Gilmore, one of the pioneers of Sun Microsystems, immediately offered
-his own extensive financial and personal support. Gilmore, an ardent
-libertarian, was to prove an eloquent advocate of electronic privacy issues,
-especially freedom from governmental and corporate computer-assisted
-surveillance of private citizens.
-
-A second meeting in San Francisco rounded up further allies:
-Stewart Brand of the Point Foundation, virtual-reality pioneers
-Jaron Lanier and Chuck Blanchard, network entrepreneur and venture
-capitalist Nat Goldhaber. At this dinner meeting, the activists settled on
-a formal title: the Electronic Frontier Foundation, Incorporated.
-Kapor became its president. A new EFF Conference was opened on
-the Point Foundation's Well, and the Well was declared
-"the home of the Electronic Frontier Foundation."
-
-Press coverage was immediate and intense. Like their
-nineteenth-century spiritual ancestors, Alexander Graham Bell
-and Thomas Watson, the high-tech computer entrepreneurs
-of the 1970s and 1980s--people such as Wozniak, Jobs, Kapor,
-Gates, and H. Ross Perot, who had raised themselves by their bootstraps
-to dominate a glittering new industry--had always made very good copy.
-
-But while the Wellbeings rejoiced, the press in general seemed
-nonplussed by the self-declared "civilizers of cyberspace."
-EFF's insistence that the war against "hackers" involved grave
-Constitutional civil liberties issues seemed somewhat farfetched,
-especially since none of EFF's organizers were lawyers
-or established politicians. The business press in particular
-found it easier to seize on the apparent core of the story--
-that high-tech entrepreneur Mitchell Kapor had established
-a "defense fund for hackers." Was EFF a genuinely important
-political development--or merely a clique of wealthy eccentrics,
-dabbling in matters better left to the proper authorities?
-The jury was still out.
-
-But the stage was now set for open confrontation.
-And the first and the most critical battle was the
-hacker show-trial of "Knight Lightning."
-
-#
-
-It has been my practice throughout this book to refer to hackers
-only by their "handles." There is little to gain by giving
-the real names of these people, many of whom are juveniles,
-many of whom have never been convicted of any crime, and many
-of whom had unsuspecting parents who have already suffered enough.
-
-But the trial of Knight Lightning on July 24-27, 1990,
-made this particular "hacker" a nationally known public figure.
-It can do no particular harm to himself or his family if I repeat
-the long-established fact that his name is Craig Neidorf (pronounced NYE-dorf).
-
-Neidorf's jury trial took place in the United States District Court,
-Northern District of Illinois, Eastern Division, with the
-Honorable Nicholas J. Bua presiding. The United States of America
-was the plaintiff, the defendant Mr. Neidorf. The defendant's attorney
-was Sheldon T. Zenner of the Chicago firm of Katten, Muchin and Zavis.
-
-The prosecution was led by the stalwarts of the Chicago Computer Fraud
-and Abuse Task Force: William J. Cook, Colleen D. Coughlin, and
-David A. Glockner, all Assistant United States Attorneys.
-The Secret Service Case Agent was Timothy M. Foley.
-
-It will be recalled that Neidorf was the co-editor of an underground hacker
-"magazine" called Phrack. Phrack was an entirely electronic publication,
-distributed through bulletin boards and over electronic networks.
-It was amateur publication given away for free. Neidorf had never made
-any money for his work in Phrack. Neither had his unindicted co-editor
-"Taran King" or any of the numerous Phrack contributors.
-
-The Chicago Computer Fraud and Abuse Task Force, however,
-had decided to prosecute Neidorf as a fraudster.
-To formally admit that Phrack was a "magazine"
-and Neidorf a "publisher" was to open a prosecutorial
-Pandora's Box of First Amendment issues. To do this
-was to play into the hands of Zenner and his EFF advisers,
-which now included a phalanx of prominent New York civil rights
-lawyers as well as the formidable legal staff of Katten, Muchin and Zavis.
-Instead, the prosecution relied heavily on the issue of access device fraud:
-Section 1029 of Title 18, the section from which the Secret Service drew
-its most direct jurisdiction over computer crime.
-
-Neidorf's alleged crimes centered around the E911 Document.
-He was accused of having entered into a fraudulent scheme with the Prophet,
-who, it will be recalled, was the Atlanta LoD member who had illicitly
-copied the E911 Document from the BellSouth AIMSX system.
-
-The Prophet himself was also a co-defendant in the Neidorf case,
-part-and-parcel of the alleged "fraud scheme" to "steal" BellSouth's
-E911 Document (and to pass the Document across state lines,
-which helped establish the Neidorf trial as a federal case).
-The Prophet, in the spirit of full co-operation, had agreed
-to testify against Neidorf.
-
-In fact, all three of the Atlanta crew stood ready to testify against Neidorf.
-Their own federal prosecutors in Atlanta had charged the Atlanta Three with:
-(a) conspiracy, (b) computer fraud, (c) wire fraud, (d) access device fraud,
-and (e) interstate transportation of stolen property (Title 18, Sections 371,
-1030, 1343, 1029, and 2314).
-
-Faced with this blizzard of trouble, Prophet and Leftist had ducked
-any public trial and had pled guilty to reduced charges--one conspiracy
-count apiece. Urvile had pled guilty to that odd bit of Section 1029
-which makes it illegal to possess "fifteen or more" illegal access devices
-(in his case, computer passwords). And their sentences were scheduled
-for September 14, 1990--well after the Neidorf trial. As witnesses,
-they could presumably be relied upon to behave.
-
-Neidorf, however, was pleading innocent. Most everyone else caught up
-in the crackdown had "cooperated fully" and pled guilty in hope
-of reduced sentences. (Steve Jackson was a notable exception,
-of course, and had strongly protested his innocence from the
-very beginning. But Steve Jackson could not get a day in court--
-Steve Jackson had never been charged with any crime in the first place.)
-
-Neidorf had been urged to plead guilty. But Neidorf was a political science
-major and was disinclined to go to jail for "fraud" when he had not made
-any money, had not broken into any computer, and had been publishing
-a magazine that he considered protected under the First Amendment.
-
-Neidorf's trial was the ONLY legal action of the entire Crackdown
-that actually involved bringing the issues at hand out for a public test
-in front of a jury of American citizens.
-
-Neidorf, too, had cooperated with investigators. He had voluntarily
-handed over much of the evidence that had led to his own indictment.
-He had already admitted in writing that he knew that the E911 Document
-had been stolen before he had "published" it in Phrack--or, from the
-prosecution's point of view, illegally transported stolen property by wire
-in something purporting to be a "publication."
-
-But even if the "publication" of the E911 Document was not held to be a crime,
-that wouldn't let Neidorf off the hook. Neidorf had still received
-the E911 Document when Prophet had transferred it to him from Rich Andrews'
-Jolnet node. On that occasion, it certainly hadn't been "published"--
-it was hacker booty, pure and simple, transported across state lines.
-
-The Chicago Task Force led a Chicago grand jury to indict Neidorf
-on a set of charges that could have put him in jail for thirty years.
-When some of these charges were successfully challenged before Neidorf
-actually went to trial, the Chicago Task Force rearranged his
-indictment so that he faced a possible jail term of over sixty years!
-As a first offender, it was very unlikely that Neidorf would in fact
-receive a sentence so drastic; but the Chicago Task Force clearly
-intended to see Neidorf put in prison, and his conspiratorial "magazine"
-put permanently out of commission. This was a federal case, and Neidorf
-was charged with the fraudulent theft of property worth almost
-eighty thousand dollars.
-
-William Cook was a strong believer in high-profile prosecutions
-with symbolic overtones. He often published articles on his work
-in the security trade press, arguing that "a clear message had
-to be sent to the public at large and the computer community
-in particular that unauthorized attacks on computers and the theft
-of computerized information would not be tolerated by the courts."
-
-The issues were complex, the prosecution's tactics somewhat unorthodox,
-but the Chicago Task Force had proved sure-footed to date. "Shadowhawk"
-had been bagged on the wing in 1989 by the Task Force, and sentenced
-to nine months in prison, and a $10,000 fine. The Shadowhawk case involved
-charges under Section 1030, the "federal interest computer" section.
-
-Shadowhawk had not in fact been a devotee of "federal-interest" computers
-per se. On the contrary, Shadowhawk, who owned an AT&T home computer,
-seemed to cherish a special aggression toward AT&T. He had bragged on
-the underground boards "Phreak Klass 2600" and "Dr. Ripco" of his skills
-at raiding AT&T, and of his intention to crash AT&T's national phone system.
-Shadowhawk's brags were noticed by Henry Kluepfel of Bellcore Security,
-scourge of the outlaw boards, whose relations with the Chicago Task Force
-were long and intimate.
-
-The Task Force successfully established that Section 1030 applied to
-the teenage Shadowhawk, despite the objections of his defense attorney.
-Shadowhawk had entered a computer "owned" by U.S. Missile Command
-and merely "managed" by AT&T. He had also entered an AT&T computer
-located at Robbins Air Force Base in Georgia. Attacking AT&T was
-of "federal interest" whether Shadowhawk had intended it or not.
-
-The Task Force also convinced the court that a piece of AT&T
-software that Shadowhawk had illicitly copied from Bell Labs,
-the "Artificial Intelligence C5 Expert System," was worth a cool
-one million dollars. Shadowhawk's attorney had argued that
-Shadowhawk had not sold the program and had made no profit from
-the illicit copying. And in point of fact, the C5 Expert System
-was experimental software, and had no established market value
-because it had never been on the market in the first place.
-AT&T's own assessment of a "one million dollar" figure for its
-own intangible property was accepted without challenge
-by the court, however. And the court concurred with
-the government prosecutors that Shadowhawk showed clear
-"intent to defraud" whether he'd gotten any money or not.
-Shadowhawk went to jail.
-
-The Task Force's other best-known triumph had been the conviction
-and jailing of "Kyrie." Kyrie, a true denizen of the digital
-criminal underground, was a 36-year-old Canadian woman,
-convicted and jailed for telecommunications fraud in Canada.
-After her release from prison, she had fled the wrath of Canada Bell
-and the Royal Canadian Mounted Police, and eventually settled,
-very unwisely, in Chicago.
-
-"Kyrie," who also called herself "Long Distance Information,"
-specialized in voice-mail abuse. She assembled large numbers
-of hot long-distance codes, then read them aloud into a series
-of corporate voice-mail systems. Kyrie and her friends were
-electronic squatters in corporate voice-mail systems,
-using them much as if they were pirate bulletin boards,
-then moving on when their vocal chatter clogged the system
-and the owners necessarily wised up. Kyrie's camp followers
-were a loose tribe of some hundred and fifty phone-phreaks,
-who followed her trail of piracy from machine to machine,
-ardently begging for her services and expertise.
-
-Kyrie's disciples passed her stolen credit-card numbers,
-in exchange for her stolen "long distance information."
-Some of Kyrie's clients paid her off in cash, by scamming
-credit-card cash advances from Western Union.
-
-Kyrie travelled incessantly, mostly through airline tickets
-and hotel rooms that she scammed through stolen credit cards.
-Tiring of this, she found refuge with a fellow female phone
-phreak in Chicago. Kyrie's hostess, like a surprising number
-of phone phreaks, was blind. She was also physically disabled.
-Kyrie allegedly made the best of her new situation by applying for,
-and receiving, state welfare funds under a false identity as
-a qualified caretaker for the handicapped.
-
-Sadly, Kyrie's two children by a former marriage had also vanished
-underground with her; these pre-teen digital refugees had no legal
-American identity, and had never spent a day in school.
-
-Kyrie was addicted to technical mastery and enthralled by her own
-cleverness and the ardent worship of her teenage followers.
-This foolishly led her to phone up Gail Thackeray in Arizona,
-to boast, brag, strut, and offer to play informant.
-Thackeray, however, had already learned far more
-than enough about Kyrie, whom she roundly despised
-as an adult criminal corrupting minors, a "female Fagin."
-Thackeray passed her tapes of Kyrie's boasts to the Secret Service.
-
-Kyrie was raided and arrested in Chicago in May 1989.
-She confessed at great length and pled guilty.
-
-In August 1990, Cook and his Task Force colleague Colleen Coughlin
-sent Kyrie to jail for 27 months, for computer and telecommunications fraud.
-This was a markedly severe sentence by the usual wrist-slapping standards
-of "hacker" busts. Seven of Kyrie's foremost teenage disciples were also
-indicted and convicted. The Kyrie "high-tech street gang," as Cook
-described it, had been crushed. Cook and his colleagues had been
-the first ever to put someone in prison for voice-mail abuse.
-Their pioneering efforts had won them attention and kudos.
-
-In his article on Kyrie, Cook drove the message home to the readers
-of Security Management magazine, a trade journal for corporate
-security professionals. The case, Cook said, and Kyrie's stiff sentence,
-"reflect a new reality for hackers and computer crime victims in the
-'90s. . . . Individuals and corporations who report computer
-and telecommunications crimes can now expect that their cooperation
-with federal law enforcement will result in meaningful punishment.
-Companies and the public at large must report computer-enhanced
-crimes if they want prosecutors and the course to protect their rights
-to the tangible and intangible property developed and stored on computers."
-
-Cook had made it his business to construct this "new reality for hackers."
-He'd also made it his business to police corporate property rights
-to the intangible.
-
-Had the Electronic Frontier Foundation been a "hacker defense fund"
-as that term was generally understood, they presumably would have stood up
-for Kyrie. Her 1990 sentence did indeed send a "message" that federal heat
-was coming down on "hackers." But Kyrie found no defenders at EFF,
-or anywhere else, for that matter. EFF was not a bail-out fund
-for electronic crooks.
-
-The Neidorf case paralleled the Shadowhawk case in certain ways.
-The victim once again was allowed to set the value of the "stolen" property.
-Once again Kluepfel was both investigator and technical advisor.
-Once again no money had changed hands, but the "intent to defraud" was central.
-
-The prosecution's case showed signs of weakness early on. The Task Force
-had originally hoped to prove Neidorf the center of a nationwide
-Legion of Doom criminal conspiracy. The Phrack editors threw physical
-get-togethers every summer, which attracted hackers from across the country;
-generally two dozen or so of the magazine's favorite contributors and readers.
-(Such conventions were common in the hacker community; 2600 Magazine,
-for instance, held public meetings of hackers in New York, every month.)
-LoD heavy-dudes were always a strong presence at these Phrack-sponsored
-"Summercons."
-
-In July 1988, an Arizona hacker named "Dictator" attended Summercon
-in Neidorf's home town of St. Louis. Dictator was one of Gail Thackeray's
-underground informants; Dictator's underground board in Phoenix was
-a sting operation for the Secret Service. Dictator brought an undercover
-crew of Secret Service agents to Summercon. The agents bored spyholes
-through the wall of Dictator's hotel room in St Louis, and videotaped
-the frolicking hackers through a one-way mirror. As it happened,
-however, nothing illegal had occurred on videotape, other than the
-guzzling of beer by a couple of minors. Summercons were social events,
-not sinister cabals. The tapes showed fifteen hours of raucous laughter,
-pizza-gobbling, in-jokes and back-slapping.
-
-Neidorf's lawyer, Sheldon Zenner, saw the Secret Service tapes
-before the trial. Zenner was shocked by the complete harmlessness
-of this meeting, which Cook had earlier characterized as a sinister
-interstate conspiracy to commit fraud. Zenner wanted to show the
-Summercon tapes to the jury. It took protracted maneuverings
-by the Task Force to keep the tapes from the jury as "irrelevant."
-
-The E911 Document was also proving a weak reed. It had originally
-been valued at $79,449. Unlike Shadowhawk's arcane Artificial Intelligence
-booty, the E911 Document was not software--it was written in English.
-Computer-knowledgeable people found this value--for a twelve-page
-bureaucratic document--frankly incredible. In his "Crime and Puzzlement"
-manifesto for EFF, Barlow commented: "We will probably never know how
-this figure was reached or by whom, though I like to imagine an appraisal
-team consisting of Franz Kafka, Joseph Heller, and Thomas Pynchon."
-
-As it happened, Barlow was unduly pessimistic. The EFF did, in fact,
-eventually discover exactly how this figure was reached, and by whom--
-but only in 1991, long after the Neidorf trial was over.
-
-Kim Megahee, a Southern Bell security manager,
-had arrived at the document's value by simply adding up
-the "costs associated with the production" of the E911 Document.
-Those "costs" were as follows:
-
-1. A technical writer had been hired to research and write the E911 Document.
- 200 hours of work, at $35 an hour, cost : $7,000. A Project Manager had
- overseen the technical writer. 200 hours, at $31 an hour, made: $6,200.
-
-2. A week of typing had cost $721 dollars. A week of formatting had
- cost $721. A week of graphics formatting had cost $742.
-
-3. Two days of editing cost $367.
-
-4. A box of order labels cost five dollars.
-
-5. Preparing a purchase order for the Document, including typing
- and the obtaining of an authorizing signature from within the
- BellSouth bureaucracy, cost $129.
-
-6. Printing cost $313. Mailing the Document to fifty people
- took fifty hours by a clerk, and cost $858.
-
-7. Placing the Document in an index took two clerks an hour each,
- totalling $43.
-
-Bureaucratic overhead alone, therefore, was alleged to have cost
-a whopping $17,099. According to Mr. Megahee, the typing
-of a twelve-page document had taken a full week. Writing it
-had taken five weeks, including an overseer who apparently
-did nothing else but watch the author for five weeks.
-Editing twelve pages had taken two days. Printing and mailing
-an electronic document (which was already available on the
-Southern Bell Data Network to any telco employee who needed it),
-had cost over a thousand dollars.
-
-But this was just the beginning. There were also the HARDWARE EXPENSES.
-Eight hundred fifty dollars for a VT220 computer monitor.
-THIRTY-ONE THOUSAND DOLLARS for a sophisticated VAXstation II computer.
-Six thousand dollars for a computer printer. TWENTY-TWO THOUSAND DOLLARS
-for a copy of "Interleaf" software. Two thousand five hundred dollars
-for VMS software. All this to create the twelve-page Document.
-
-Plus ten percent of the cost of the software and the hardware, for maintenance.
-(Actually, the ten percent maintenance costs, though mentioned, had been left
-off the final $79,449 total, apparently through a merciful oversight).
-
-Mr. Megahee's letter had been mailed directly to William Cook himself,
-at the office of the Chicago federal attorneys. The United States Government
-accepted these telco figures without question.
-
-As incredulity mounted, the value of the E911 Document was officially
-revised downward. This time, Robert Kibler of BellSouth Security
-estimated the value of the twelve pages as a mere $24,639.05--based,
-purportedly, on "R&D costs." But this specific estimate,
-right down to the nickel, did not move the skeptics at all;
-in fact it provoked open scorn and a torrent of sarcasm.
-
-The financial issues concerning theft of proprietary information
-have always been peculiar. It could be argued that BellSouth
-had not "lost" its E911 Document at all in the first place,
-and therefore had not suffered any monetary damage from this "theft."
-And Sheldon Zenner did in fact argue this at Neidorf's trial--
-that Prophet's raid had not been "theft," but was better understood
-as illicit copying.
-
-The money, however, was not central to anyone's true purposes in this trial.
-It was not Cook's strategy to convince the jury that the E911 Document
-was a major act of theft and should be punished for that reason alone.
-His strategy was to argue that the E911 Document was DANGEROUS.
-It was his intention to establish that the E911 Document was "a road-map"
-to the Enhanced 911 System. Neidorf had deliberately and recklessly
-distributed a dangerous weapon. Neidorf and the Prophet did not care
-(or perhaps even gloated at the sinister idea) that the E911 Document
-could be used by hackers to disrupt 911 service, "a life line for every
-person certainly in the Southern Bell region of the United States,
-and indeed, in many communities throughout the United States,"
-in Cook's own words. Neidorf had put people's lives in danger.
-
-In pre-trial maneuverings, Cook had established that the E911 Document
-was too hot to appear in the public proceedings of the Neidorf trial.
-The JURY ITSELF would not be allowed to ever see this Document,
-lest it slip into the official court records, and thus into the hands
-of the general public, and, thus, somehow, to malicious hackers
-who might lethally abuse it.
-
-Hiding the E911 Document from the jury may have been a
-clever legal maneuver, but it had a severe flaw. There were,
-in point of fact, hundreds, perhaps thousands, of people,
-already in possession of the E911 Document, just as Phrack
-had published it. Its true nature was already obvious
-to a wide section of the interested public (all of whom,
-by the way, were, at least theoretically, party to
-a gigantic wire-fraud conspiracy). Most everyone
-in the electronic community who had a modem and any
-interest in the Neidorf case already had a copy of the Document.
-It had already been available in Phrack for over a year.
-
-People, even quite normal people without any particular
-prurient interest in forbidden knowledge, did not shut their eyes
-in terror at the thought of beholding a "dangerous" document
-from a telephone company. On the contrary, they tended to trust
-their own judgement and simply read the Document for themselves.
-And they were not impressed.
-
-One such person was John Nagle. Nagle was a forty-one-year-old
-professional programmer with a masters' degree in computer science
-from Stanford. He had worked for Ford Aerospace, where he had invented
-a computer-networking technique known as the "Nagle Algorithm,"
-and for the prominent Californian computer-graphics firm "Autodesk,"
-where he was a major stockholder.
-
-Nagle was also a prominent figure on the Well, much respected
-for his technical knowledgeability.
-
-Nagle had followed the civil-liberties debate closely,
-for he was an ardent telecommunicator. He was no particular friend
-of computer intruders, but he believed electronic publishing
-had a great deal to offer society at large, and attempts
-to restrain its growth, or to censor free electronic expression,
-strongly roused his ire.
-
-The Neidorf case, and the E911 Document, were both being discussed
-in detail on the Internet, in an electronic publication called Telecom Digest.
-Nagle, a longtime Internet maven, was a regular reader of Telecom Digest.
-Nagle had never seen a copy of Phrack, but the implications of the case
-disturbed him.
-
-While in a Stanford bookstore hunting books on robotics,
-Nagle happened across a book called The Intelligent Network.
-Thumbing through it at random, Nagle came across an entire chapter
-meticulously detailing the workings of E911 police emergency systems.
-This extensive text was being sold openly, and yet in Illinois
-a young man was in danger of going to prison for publishing
-a thin six-page document about 911 service.
-
-Nagle made an ironic comment to this effect in Telecom Digest.
-From there, Nagle was put in touch with Mitch Kapor,
-and then with Neidorf's lawyers.
-
-Sheldon Zenner was delighted to find a computer telecommunications expert
-willing to speak up for Neidorf, one who was not a wacky teenage "hacker."
-Nagle was fluent, mature, and respectable; he'd once had a federal
-security clearance.
-
-Nagle was asked to fly to Illinois to join the defense team.
-
-Having joined the defense as an expert witness, Nagle read the entire
-E911 Document for himself. He made his own judgement about its potential
-for menace.
-
-The time has now come for you yourself, the reader, to have a look
-at the E911 Document. This six-page piece of work was the pretext
-for a federal prosecution that could have sent an electronic publisher
-to prison for thirty, or even sixty, years. It was the pretext
-for the search and seizure of Steve Jackson Games, a legitimate publisher
-of printed books. It was also the formal pretext for the search
-and seizure of the Mentor's bulletin board, "Phoenix Project,"
-and for the raid on the home of Erik Bloodaxe. It also had much
-to do with the seizure of Richard Andrews' Jolnet node
-and the shutdown of Charles Boykin's AT&T node.
-The E911 Document was the single most important piece
-of evidence in the Hacker Crackdown. There can be no real
-and legitimate substitute for the Document itself.
-
-
-==Phrack Inc.==
-
-Volume Two, Issue 24, File 5 of 13
-
-Control Office Administration
-Of Enhanced 911 Services For
-Special Services and Account Centers
-
-by the Eavesdropper
-
-March, 1988
-
-
-Description of Service
-~~~~~~~~~~~~~~~~~~~~~
-The control office for Emergency 911 service is assigned in
-accordance with the existing standard guidelines to one of
-the following centers:
-
-o Special Services Center (SSC)
-o Major Accounts Center (MAC)
-o Serving Test Center (STC)
-o Toll Control Center (TCC)
-
-The SSC/MAC designation is used in this document interchangeably
-for any of these four centers. The Special Services Centers (SSCs)
-or Major Account Centers (MACs) have been designated as the trouble
-reporting contact for all E911 customer (PSAP) reported troubles.
-Subscribers who have trouble on an E911 call will continue
-to contact local repair service (CRSAB) who will refer the
-trouble to the SSC/MAC, when appropriate.
-
-Due to the critical nature of E911 service, the control
-and timely repair of troubles is demanded. As the primary
-E911 customer contact, the SSC/MAC is in the unique position
-to monitor the status of the trouble and insure its resolution.
-
-System Overview
-~~~~~~~~~~~~~~
-The number 911 is intended as a nationwide universal
-telephone number which provides the public with direct
-access to a Public Safety Answering Point (PSAP). A PSAP
-is also referred to as an Emergency Service Bureau (ESB).
-A PSAP is an agency or facility which is authorized by a
-municipality to receive and respond to police, fire and/or
-ambulance services. One or more attendants are located
-at the PSAP facilities to receive and handle calls of an
-emergency nature in accordance with the local municipal
-requirements.
-
-An important advantage of E911 emergency service is
-improved (reduced) response times for emergency
-services. Also close coordination among agencies
-providing various emergency services is a valuable
-capability provided by E911 service.
-
-1A ESS is used as the tandem office for the E911 network to
-route all 911 calls to the correct (primary) PSAP designated
-to serve the calling station. The E911 feature was
-developed primarily to provide routing to the correct PSAP
-for all 911 calls. Selective routing allows a 911 call
-originated from a particular station located in a particular
-district, zone, or town, to be routed to the primary PSAP
-designated to serve that customer station regardless of
-wire center boundaries. Thus, selective routing eliminates
-the problem of wire center boundaries not coinciding with
-district or other political boundaries.
-
-The services available with the E911 feature include:
-
-Forced Disconnect Default Routing
-Alternative Routing Night Service
-Selective Routing Automatic Number
-Identification (ANI)
-Selective Transfer Automatic Location
-Identification (ALI)
-
-
-Preservice/Installation Guidelines
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-When a contract for an E911 system has been signed, it is
-the responsibility of Network Marketing to establish an
-implementation/cutover committee which should include
-a representative from the SSC/MAC. Duties of the E911
-Implementation Team include coordination of all phases
-of the E911 system deployment and the formation of an
-on-going E911 maintenance subcommittee.
-
-Marketing is responsible for providing the following
-customer specific information to the SSC/MAC prior to
-the start of call through testing:
-
-o All PSAP's (name, address, local contact)
-o All PSAP circuit ID's
-o 1004 911 service request including PSAP details on each PSAP
- (1004 Section K, L, M)
-o Network configuration
-o Any vendor information (name, telephone number, equipment)
-
-The SSC/MAC needs to know if the equipment and sets
-at the PSAP are maintained by the BOCs, an independent
-company, or an outside vendor, or any combination.
-This information is then entered on the PSAP profile sheets
-and reviewed quarterly for changes, additions and deletions.
-
-Marketing will secure the Major Account Number (MAN)
-and provide this number to Corporate Communications
-so that the initial issue of the service orders carry
-the MAN and can be tracked by the SSC/MAC via CORDNET.
-PSAP circuits are official services by definition.
-
-All service orders required for the installation of the E911
-system should include the MAN assigned to the city/county
-which has purchased the system.
-
-In accordance with the basic SSC/MAC strategy for provisioning,
-the SSC/MAC will be Overall Control Office (OCO) for all Node
-to PSAP circuits (official services) and any other services
-for this customer. Training must be scheduled for all SSC/MAC
-involved personnel during the pre-service stage of the project.
-
-The E911 Implementation Team will form the on-going
-maintenance subcommittee prior to the initial
-implementation of the E911 system. This sub-committee
-will establish post implementation quality assurance
-procedures to ensure that the E911 system continues to
-provide quality service to the customer.
-Customer/Company training, trouble reporting interfaces
-for the customer, telephone company and any involved
-independent telephone companies needs to be addressed
-and implemented prior to E911 cutover. These functions
-can be best addressed by the formation of a sub-
-committee of the E911 Implementation Team to set up
-guidelines for and to secure service commitments of
-interfacing organizations. A SSC/MAC supervisor should
-chair this subcommittee and include the following
-organizations:
-
-1) Switching Control Center
- - E911 translations
- - Trunking
- - End office and Tandem office hardware/software
-2) Recent Change Memory Administration Center
- - Daily RC update activity for TN/ESN translations
- - Processes validity errors and rejects
-3) Line and Number Administration
- - Verification of TN/ESN translations
-4) Special Service Center/Major Account Center
- - Single point of contact for all PSAP and Node to host troubles
- - Logs, tracks & statusing of all trouble reports
- - Trouble referral, follow up, and escalation
- - Customer notification of status and restoration
- - Analyzation of "chronic" troubles
- - Testing, installation and maintenance of E911 circuits
-5) Installation and Maintenance (SSIM/I&M)
- - Repair and maintenance of PSAP equipment and Telco owned sets
-6) Minicomputer Maintenance Operations Center
- - E911 circuit maintenance (where applicable)
-7) Area Maintenance Engineer
- - Technical assistance on voice (CO-PSAP) network related E911 troubles
-
-
-Maintenance Guidelines
-~~~~~~~~~~~~~~~~~~~~~
-The CCNC will test the Node circuit from the 202T at the
-Host site to the 202T at the Node site. Since Host to Node
-(CCNC to MMOC) circuits are official company services,
-the CCNC will refer all Node circuit troubles to the
-SSC/MAC. The SSC/MAC is responsible for the testing
-and follow up to restoration of these circuit troubles.
-
-Although Node to PSAP circuit are official services, the
-MMOC will refer PSAP circuit troubles to the appropriate
-SSC/MAC. The SSC/MAC is responsible for testing and
-follow up to restoration of PSAP circuit troubles.
-
-The SSC/MAC will also receive reports from
-CRSAB/IMC(s) on subscriber 911 troubles when they are
-not line troubles. The SSC/MAC is responsible for testing
-and restoration of these troubles.
-
-Maintenance responsibilities are as follows:
-
-SCC@ Voice Network (ANI to PSAP)
-@SCC responsible for tandem switch
-
-SSIM/I&M PSAP Equipment (Modems, CIU's, sets)
-Vendor PSAP Equipment (when CPE)
-SSC/MAC PSAP to Node circuits, and tandem to
- PSAP voice circuits (EMNT)
-MMOC Node site (Modems, cables, etc)
-
-Note: All above work groups are required to resolve troubles
-by interfacing with appropriate work groups for resolution.
-
-The Switching Control Center (SCC) is responsible for
-E911/1AESS translations in tandem central offices.
-These translations route E911 calls, selective transfer,
-default routing, speed calling, etc., for each PSAP.
-The SCC is also responsible for troubleshooting on
-the voice network (call originating to end office tandem equipment).
-
-For example, ANI failures in the originating offices would
-be a responsibility of the SCC.
-
-Recent Change Memory Administration Center (RCMAC) performs
-the daily tandem translation updates (recent change)
-for routing of individual telephone numbers.
-
-Recent changes are generated from service order activity
-(new service, address changes, etc.) and compiled into
-a daily file by the E911 Center (ALI/DMS E911 Computer).
-
-SSIM/I&M is responsible for the installation and repair of
-PSAP equipment. PSAP equipment includes ANI Controller,
-ALI Controller, data sets, cables, sets, and other peripheral
-equipment that is not vendor owned. SSIM/I&M is responsible
-for establishing maintenance test kits, complete with spare parts
-for PSAP maintenance. This includes test gear, data sets,
-and ANI/ALI Controller parts.
-
-Special Services Center (SSC) or Major Account Center
-(MAC) serves as the trouble reporting contact for all
-(PSAP) troubles reported by customer. The SSC/MAC
-refers troubles to proper organizations for handling and
-tracks status of troubles, escalating when necessary.
-The SSC/MAC will close out troubles with customer.
-The SSC/MAC will analyze all troubles and tracks "chronic"
-PSAP troubles.
-
-Corporate Communications Network Center (CCNC) will
-test and refer troubles on all node to host circuits.
-All E911 circuits are classified as official company property.
-
-The Minicomputer Maintenance Operations Center
-(MMOC) maintains the E911 (ALI/DMS) computer
-hardware at the Host site. This MMOC is also responsible
-for monitoring the system and reporting certain PSAP
-and system problems to the local MMOC's, SCC's or
-SSC/MAC's. The MMOC personnel also operate software
-programs that maintain the TN data base under the
-direction of the E911 Center. The maintenance of the
-NODE computer (the interface between the PSAP and the
-ALI/DMS computer) is a function of the MMOC at the
-NODE site. The MMOC's at the NODE sites may also be
-involved in the testing of NODE to Host circuits.
-The MMOC will also assist on Host to PSAP and data network
-related troubles not resolved through standard trouble
-clearing procedures.
-
-Installation And Maintenance Center (IMC) is responsible
-for referral of E911 subscriber troubles that are not subscriber
-line problems.
-
-E911 Center - Performs the role of System Administration
-and is responsible for overall operation of the E911
-computer software. The E911 Center does A-Z trouble
-analysis and provides statistical information on the
-performance of the system.
-
-This analysis includes processing PSAP inquiries (trouble
-reports) and referral of network troubles. The E911 Center
-also performs daily processing of tandem recent change
-and provides information to the RCMAC for tandem input.
-The E911 Center is responsible for daily processing
-of the ALI/DMS computer data base and provides error files,
-etc. to the Customer Services department for investigation and correction.
-The E911 Center participates in all system implementations and on-going
-maintenance effort and assists in the development of procedures,
-training and education of information to all groups.
-
-Any group receiving a 911 trouble from the SSC/MAC should
-close out the trouble with the SSC/MAC or provide a status
-if the trouble has been referred to another group.
-This will allow the SSC/MAC to provide a status back
-to the customer or escalate as appropriate.
-
-Any group receiving a trouble from the Host site (MMOC
-or CCNC) should close the trouble back to that group.
-
-The MMOC should notify the appropriate SSC/MAC
-when the Host, Node, or all Node circuits are down so that
-the SSC/MAC can reply to customer reports that may be
-called in by the PSAPs. This will eliminate duplicate
-reporting of troubles. On complete outages the MMOC
-will follow escalation procedures for a Node after two (2)
-hours and for a PSAP after four (4) hours. Additionally the
-MMOC will notify the appropriate SSC/MAC when the
-Host, Node, or all Node circuits are down.
-
-The PSAP will call the SSC/MAC to report E911 troubles.
-The person reporting the E911 trouble may not have a
-circuit I.D. and will therefore report the PSAP name and
-address. Many PSAP troubles are not circuit specific. In
-those instances where the caller cannot provide a circuit
-I.D., the SSC/MAC will be required to determine the
-circuit I.D. using the PSAP profile. Under no circumstances
-will the SSC/MAC Center refuse to take the trouble.
-The E911 trouble should be handled as quickly as possible,
-with the SSC/MAC providing as much assistance as
-possible while taking the trouble report from the caller.
-
-The SSC/MAC will screen/test the trouble to determine the
-appropriate handoff organization based on the following criteria:
-
-PSAP equipment problem: SSIM/I&M
-Circuit problem: SSC/MAC
-Voice network problem: SCC (report trunk group number)
-Problem affecting multiple PSAPs (No ALI report from
-all PSAPs): Contact the MMOC to check for NODE or
-Host computer problems before further testing.
-
-The SSC/MAC will track the status of reported troubles
-and escalate as appropriate. The SSC/MAC will close out
-customer/company reports with the initiating contact.
-Groups with specific maintenance responsibilities,
-defined above, will investigate "chronic" troubles upon
-request from the SSC/MAC and the ongoing maintenance subcommittee.
-
-All "out of service" E911 troubles are priority one type reports.
-One link down to a PSAP is considered a priority one trouble
-and should be handled as if the PSAP was isolated.
-
-The PSAP will report troubles with the ANI controller, ALI
-controller or set equipment to the SSC/MAC.
-
-NO ANI: Where the PSAP reports NO ANI (digital
-display screen is blank) ask if this condition exists on all
-screens and on all calls. It is important to differentiate
-between blank screens and screens displaying 911-00XX,
-or all zeroes.
-
-When the PSAP reports all screens on all calls, ask if there
-is any voice contact with callers. If there is no voice
-contact the trouble should be referred to the SCC
-immediately since 911 calls are not getting through which
-may require alternate routing of calls to another PSAP.
-
-When the PSAP reports this condition on all screens
-but not all calls and has voice contact with callers,
-the report should be referred to SSIM/I&M for dispatch.
-The SSC/MAC should verify with the SCC that ANI
-is pulsing before dispatching SSIM.
-
-When the PSAP reports this condition on one screen for
-all calls (others work fine) the trouble should be referred
-to SSIM/I&M for dispatch, because the trouble is isolated to
-one piece of equipment at the customer premise.
-
-An ANI failure (i.e. all zeroes) indicates that the ANI has
-not been received by the PSAP from the tandem office or
-was lost by the PSAP ANI controller. The PSAP may
-receive "02" alarms which can be caused by the ANI
-controller logging more than three all zero failures on the
-same trunk. The PSAP has been instructed to report this
-condition to the SSC/MAC since it could indicate an
-equipment trouble at the PSAP which might be affecting
-all subscribers calling into the PSAP. When all zeroes are
-being received on all calls or "02" alarms continue, a tester
-should analyze the condition to determine the appropriate
-action to be taken. The tester must perform cooperative
-testing with the SCC when there appears to be a problem
-on the Tandem-PSAP trunks before requesting dispatch.
-
-When an occasional all zero condition is reported,
-the SSC/MAC should dispatch SSIM/I&M to routine
-equipment on a "chronic" troublesweep.
-
-The PSAPs are instructed to report incidental ANI failures
-to the BOC on a PSAP inquiry trouble ticket (paper) that
-is sent to the Customer Services E911 group and forwarded
-to E911 center when required. This usually involves only a
-particular telephone number and is not a condition that
-would require a report to the SSC/MAC. Multiple ANI
-failures which our from the same end office (XX denotes
-end office), indicate a hard trouble condition may exist
-in the end office or end office tandem trunks. The PSAP will
-report this type of condition to the SSC/MAC and the
-SSC/MAC should refer the report to the SCC responsible
-for the tandem office. NOTE: XX is the ESCO (Emergency
-Service Number) associated with the incoming 911 trunks
-into the tandem. It is important that the C/MAC tell the
-SCC what is displayed at the PSAP (i.e. 911-0011) which
-indicates to the SCC which end office is in trouble.
-
-Note: It is essential that the PSAP fill out inquiry form
-on every ANI failure.
-
-The PSAP will report a trouble any time an address is not
-received on an address display (screen blank) E911 call.
-(If a record is not in the 911 data base or an ANI failure
-is encountered, the screen will provide a display noticing
-such condition). The SSC/MAC should verify with the PSAP
-whether the NO ALI condition is on one screen or all screens.
-
-When the condition is on one screen (other screens
-receive ALI information) the SSC/MAC will request
-SSIM/I&M to dispatch.
-
-If no screens are receiving ALI information, there is usually
-a circuit trouble between the PSAP and the Host computer.
-The SSC/MAC should test the trouble and refer for restoral.
-
-Note: If the SSC/MAC receives calls from multiple
-PSAP's, all of which are receiving NO ALI, there is a
-problem with the Node or Node to Host circuits or the
-Host computer itself. Before referring the trouble the
-SSC/MAC should call the MMOC to inquire if the Node
-or Host is in trouble.
-
-Alarm conditions on the ANI controller digital display at
-the PSAP are to be reported by the PSAP's. These alarms
-can indicate various trouble conditions so the SSC/MAC
-should ask the PSAP if any portion of the E911 system
-is not functioning properly.
-
-The SSC/MAC should verify with the PSAP attendant that
-the equipment's primary function is answering E911 calls.
-If it is, the SSC/MAC should request a dispatch SSIM/I&M.
-If the equipment is not primarily used for E911,
-then the SSC/MAC should advise PSAP to contact their CPE vendor.
-
-Note: These troubles can be quite confusing when the
-PSAP has vendor equipment mixed in with equipment
-that the BOC maintains. The Marketing representative
-should provide the SSC/MAC information concerning any
-unusual or exception items where the PSAP should
-contact their vendor. This information should be included
-in the PSAP profile sheets.
-
-ANI or ALI controller down: When the host computer sees
-the PSAP equipment down and it does not come back up,
-the MMOC will report the trouble to the SSC/MAC;
-the equipment is down at the PSAP, a dispatch will be required.
-
-PSAP link (circuit) down: The MMOC will provide the
-SSC/MAC with the circuit ID that the Host computer
-indicates in trouble. Although each PSAP has two circuits,
-when either circuit is down the condition must be treated
-as an emergency since failure of the second circuit will
-cause the PSAP to be isolated.
-
-Any problems that the MMOC identifies from the Node
-location to the Host computer will be handled directly
-with the appropriate MMOC(s)/CCNC.
-
-Note: The customer will call only when a problem is
-apparent to the PSAP. When only one circuit is down to
-the PSAP, the customer may not be aware there is a
-trouble, even though there is one link down,
-notification should appear on the PSAP screen.
-Troubles called into the SSC/MAC from the MMOC
-or other company employee should not be closed out
-by calling the PSAP since it may result in the
-customer responding that they do not have a trouble.
-These reports can only be closed out by receiving
-information that the trouble was fixed and by checking
-with the company employee that reported the trouble.
-The MMOC personnel will be able to verify that the
-trouble has cleared by reviewing a printout from the host.
-
-When the CRSAB receives a subscriber complaint
-(i.e., cannot dial 911) the RSA should obtain as much
-information as possible while the customer is on the line.
-
-For example, what happened when the subscriber dialed 911?
-The report is automatically directed to the IMC for subscriber line testing.
-When no line trouble is found, the IMC will refer the trouble condition
-to the SSC/MAC. The SSC/MAC will contact Customer Services E911 Group
-and verify that the subscriber should be able to call 911 and obtain the ESN.
-The SSC/MAC will verify the ESN via 2SCCS. When both verifications match,
-the SSC/MAC will refer the report to the SCC responsible for the 911 tandem
-office for investigation and resolution. The MAC is responsible for tracking
-the trouble and informing the IMC when it is resolved.
-
-
-For more information, please refer to E911 Glossary of Terms.
-End of Phrack File
-_____________________________________
-
-
-The reader is forgiven if he or she was entirely unable to read
-this document. John Perry Barlow had a great deal of fun at its expense,
-in "Crime and Puzzlement:" "Bureaucrat-ese of surpassing opacity. . . .
-To read the whole thing straight through without entering coma requires
-either a machine or a human who has too much practice thinking like one.
-Anyone who can understand it fully and fluidly had altered his consciousness
-beyond the ability to ever again read Blake, Whitman, or Tolstoy. . . .
-the document contains little of interest to anyone who is not a student
-of advanced organizational sclerosis."
-
-With the Document itself to hand, however, exactly as it was published
-(in its six-page edited form) in Phrack, the reader may be able to verify
-a few statements of fact about its nature. First, there is no software,
-no computer code, in the Document. It is not computer-programming language
-like FORTRAN or C++, it is English; all the sentences have nouns and verbs
-and punctuation. It does not explain how to break into the E911 system.
-It does not suggest ways to destroy or damage the E911 system.
-
-There are no access codes in the Document. There are no computer passwords.
-It does not explain how to steal long distance service. It does not explain
-how to break in to telco switching stations. There is nothing in it about
-using a personal computer or a modem for any purpose at all, good or bad.
-
-Close study will reveal that this document is not about machinery.
-The E911 Document is about ADMINISTRATION. It describes how one creates
-and administers certain units of telco bureaucracy:
-Special Service Centers and Major Account Centers (SSC/MAC).
-It describes how these centers should distribute responsibility
-for the E911 service, to other units of telco bureaucracy,
-in a chain of command, a formal hierarchy. It describes
-who answers customer complaints, who screens calls,
-who reports equipment failures, who answers those reports,
-who handles maintenance, who chairs subcommittees,
-who gives orders, who follows orders, WHO tells WHOM what to do.
-The Document is not a "roadmap" to computers.
-The Document is a roadmap to PEOPLE.
-
-As an aid to breaking into computer systems, the Document is USELESS.
-As an aid to harassing and deceiving telco people, however, the Document
-might prove handy (especially with its Glossary, which I have not included).
-An intense and protracted study of this Document and its Glossary,
-combined with many other such documents, might teach one to speak like
-a telco employee. And telco people live by SPEECH--they live by phone
-communication. If you can mimic their language over the phone,
-you can "social-engineer" them. If you can con telco people, you can
-wreak havoc among them. You can force them to no longer trust one another;
-you can break the telephonic ties that bind their community; you can make
-them paranoid. And people will fight harder to defend their community
-than they will fight to defend their individual selves.
-
-This was the genuine, gut-level threat posed by Phrack magazine.
-The real struggle was over the control of telco language,
-the control of telco knowledge. It was a struggle to defend the social
-"membrane of differentiation" that forms the walls of the telco
-community's ivory tower --the special jargon that allows telco
-professionals to recognize one another, and to exclude charlatans,
-thieves, and upstarts. And the prosecution brought out this fact.
-They repeatedly made reference to the threat posed to telco professionals
-by hackers using "social engineering."
-
-However, Craig Neidorf was not on trial for learning to speak like
-a professional telecommunications expert. Craig Neidorf was on trial
-for access device fraud and transportation of stolen property.
-He was on trial for stealing a document that was purportedly
-highly sensitive and purportedly worth tens of thousands of dollars.
-
-#
-
-John Nagle read the E911 Document. He drew his own conclusions.
-And he presented Zenner and his defense team with an overflowing box
-of similar material, drawn mostly from Stanford University's
-engineering libraries. During the trial, the defense team--Zenner,
-half-a-dozen other attorneys, Nagle, Neidorf, and computer-security
-expert Dorothy Denning, all pored over the E911 Document line-by-line.
-
-On the afternoon of July 25, 1990, Zenner began to cross-examine
-a woman named Billie Williams, a service manager for Southern Bell
-in Atlanta. Ms. Williams had been responsible for the E911 Document.
-(She was not its author--its original "author" was a Southern Bell
-staff manager named Richard Helms. However, Mr. Helms should not bear
-the entire blame; many telco staff people and maintenance personnel
-had amended the Document. It had not been so much "written" by a
-single author, as built by committee out of concrete-blocks of jargon.)
-
-Ms. Williams had been called as a witness for the prosecution,
-and had gamely tried to explain the basic technical structure
-of the E911 system, aided by charts.
-
-Now it was Zenner's turn. He first established that the
-"proprietary stamp" that BellSouth had used on the E911 Document
-was stamped on EVERY SINGLE DOCUMENT that BellSouth wrote--
-THOUSANDS of documents. "We do not publish anything other
-than for our own company," Ms. Williams explained.
-"Any company document of this nature is considered proprietary."
-Nobody was in charge of singling out special high-security publications
-for special high-security protection. They were ALL special,
-no matter how trivial, no matter what their subject matter--
-the stamp was put on as soon as any document was written,
-and the stamp was never removed.
-
-Zenner now asked whether the charts she had been using to explain
-the mechanics of E911 system were "proprietary," too.
-Were they PUBLIC INFORMATION, these charts, all about PSAPs,
-ALIs, nodes, local end switches? Could he take the charts out
-in the street and show them to anybody, "without violating
-some proprietary notion that BellSouth has?"
-
-Ms Williams showed some confusion, but finally areed that the charts were,
-in fact, public.
-
-"But isn't this what you said was basically what appeared in Phrack?"
-
-Ms. Williams denied this.
-
-Zenner now pointed out that the E911 Document as published in Phrack
-was only half the size of the original E911 Document (as Prophet
-had purloined it). Half of it had been deleted--edited by Neidorf.
-
-Ms. Williams countered that "Most of the information that is
-in the text file is redundant."
-
-Zenner continued to probe. Exactly what bits of knowledge in the Document
-were, in fact, unknown to the public? Locations of E911 computers?
-Phone numbers for telco personnel? Ongoing maintenance subcommittees?
-Hadn't Neidorf removed much of this?
-
-Then he pounced. "Are you familiar with Bellcore Technical Reference
-Document TR-TSY-000350?" It was, Zenner explained, officially titled
-"E911 Public Safety Answering Point Interface Between 1-1AESS Switch
-and Customer Premises Equipment." It contained highly detailed
-and specific technical information about the E911 System.
-It was published by Bellcore and publicly available for about $20.
-
-He showed the witness a Bellcore catalog which listed thousands
-of documents from Bellcore and from all the Baby Bells, BellSouth included.
-The catalog, Zenner pointed out, was free. Anyone with a credit card
-could call the Bellcore toll-free 800 number and simply order any
-of these documents, which would be shipped to any customer without question.
-Including, for instance, "BellSouth E911 Service Interfaces to
-Customer Premises Equipment at a Public Safety Answering Point."
-
-Zenner gave the witness a copy of "BellSouth E911 Service Interfaces,"
-which cost, as he pointed out, $13, straight from the catalog.
-"Look at it carefully," he urged Ms. Williams, "and tell me
-if it doesn't contain about twice as much detailed information
-about the E911 system of BellSouth than appeared anywhere in Phrack."
-
-"You want me to. . . ." Ms. Williams trailed off. "I don't understand."
-
-"Take a careful look," Zenner persisted. "Take a look at that document,
-and tell me when you're done looking at it if, indeed, it doesn't contain
-much more detailed information about the E911 system than appeared in Phrack."
-
-"Phrack wasn't taken from this," Ms. Williams said.
-
-"Excuse me?" said Zenner.
-
-"Phrack wasn't taken from this."
-
-"I can't hear you," Zenner said.
-
-"Phrack was not taken from this document. I don't understand
-your question to me."
-
-"I guess you don't," Zenner said.
-
-At this point, the prosecution's case had been gutshot.
-Ms. Williams was distressed. Her confusion was quite genuine.
-Phrack had not been taken from any publicly available Bellcore document.
-Phrack's E911 Document had been stolen from her own company's computers,
-from her own company's text files, that her own colleagues had written,
-and revised, with much labor.
-
-But the "value" of the Document had been blown to smithereens.
-It wasn't worth eighty grand. According to Bellcore it was worth
-thirteen bucks. And the looming menace that it supposedly posed
-had been reduced in instants to a scarecrow. Bellcore itself
-was selling material far more detailed and "dangerous,"
-to anybody with a credit card and a phone.
-
-Actually, Bellcore was not giving this information to just anybody.
-They gave it to ANYBODY WHO ASKED, but not many did ask.
-Not many people knew that Bellcore had a free catalog and an 800 number.
-John Nagle knew, but certainly the average teenage phreak didn't know.
-"Tuc," a friend of Neidorf's and sometime Phrack contributor, knew,
-and Tuc had been very helpful to the defense, behind the scenes.
-But the Legion of Doom didn't know--otherwise, they would never
-have wasted so much time raiding dumpsters. Cook didn't know.
-Foley didn't know. Kluepfel didn't know. The right hand
-of Bellcore knew not what the left hand was doing. The right
-hand was battering hackers without mercy, while the left hand
-was distributing Bellcore's intellectual property to anybody
-who was interested in telephone technical trivia--apparently,
-a pathetic few.
-
-The digital underground was so amateurish and poorly organized
-that they had never discovered this heap of unguarded riches.
-The ivory tower of the telcos was so wrapped-up in the fog
-of its own technical obscurity that it had left all the
-windows open and flung open the doors. No one had even noticed.
-
-Zenner sank another nail in the coffin. He produced a printed issue
-of Telephone Engineer & Management, a prominent industry journal
-that comes out twice a month and costs $27 a year. This particular issue
-of TE&M, called "Update on 911," featured a galaxy of technical details
-on 911 service and a glossary far more extensive than Phrack's.
-
-The trial rumbled on, somehow, through its own momentum.
-Tim Foley testified about his interrogations of Neidorf.
-Neidorf's written admission that he had known the E911 Document
-was pilfered was officially read into the court record.
-
-An interesting side issue came up: "Terminus" had once passed Neidorf
-a piece of UNIX AT&T software, a log-in sequence, that had been cunningly
-altered so that it could trap passwords. The UNIX software itself was
-illegally copied AT&T property, and the alterations "Terminus" had made to it,
-had transformed it into a device for facilitating computer break-ins. Terminus
-himself would eventually plead guilty to theft of this piece of software,
-and the Chicago group would send Terminus to prison for it. But it was
-of dubious relevance in the Neidorf case. Neidorf hadn't written the program.
-He wasn't accused of ever having used it. And Neidorf wasn't being charged
-with software theft or owning a password trapper.
-
-On the next day, Zenner took the offensive. The civil libertarians
-now had their own arcane, untried legal weaponry to launch into action--
-the Electronic Communications Privacy Act of 1986, 18 US Code,
-Section 2701 et seq. Section 2701 makes it a crime to intentionally
-access without authorization a facility in which an electronic communication
-service is provided--it is, at heart, an anti-bugging and anti-tapping law,
-intended to carry the traditional protections of telephones into other
-electronic channels of communication. While providing penalties for amateur
-snoops, however, Section 2703 of the ECPA also lays some formal difficulties
-on the bugging and tapping activities of police.
-
-The Secret Service, in the person of Tim Foley, had served Richard Andrews
-with a federal grand jury subpoena, in their pursuit of Prophet,
-the E911 Document, and the Terminus software ring. But according to
-the Electronic Communications Privacy Act, a "provider of remote
-computing service" was legally entitled to "prior notice" from
-the government if a subpoena was used. Richard Andrews and his
-basement UNIX node, Jolnet, had not received any "prior notice."
-Tim Foley had purportedly violated the ECPA and committed
-an electronic crime! Zenner now sought the judge's permission
-to cross-examine Foley on the topic of Foley's own electronic misdeeds.
-
-Cook argued that Richard Andrews' Jolnet was a privately owned
-bulletin board, and not within the purview of ECPA. Judge Bua
-granted the motion of the government to prevent cross-examination
-on that point, and Zenner's offensive fizzled. This, however,
-was the first direct assault on the legality of the actions
-of the Computer Fraud and Abuse Task Force itself--
-the first suggestion that they themselves had broken the law,
-and might, perhaps, be called to account.
-
-Zenner, in any case, did not really need the ECPA.
-Instead, he grilled Foley on the glaring contradictions in
-the supposed value of the E911 Document. He also brought up
-the embarrassing fact that the supposedly red-hot E911 Document
-had been sitting around for months, in Jolnet, with Kluepfel's knowledge,
-while Kluepfel had done nothing about it.
-
-In the afternoon, the Prophet was brought in to testify
-for the prosecution. (The Prophet, it will be recalled,
-had also been indicted in the case as partner in a fraud
-scheme with Neidorf.) In Atlanta, the Prophet had already
-pled guilty to one charge of conspiracy, one charge of wire fraud
-and one charge of interstate transportation of stolen property.
-The wire fraud charge, and the stolen property charge,
-were both directly based on the E911 Document.
-
-The twenty-year-old Prophet proved a sorry customer,
-answering questions politely but in a barely audible mumble,
-his voice trailing off at the ends of sentences.
-He was constantly urged to speak up.
-
-Cook, examining Prophet, forced him to admit that
-he had once had a "drug problem," abusing amphetamines,
-marijuana, cocaine, and LSD. This may have established
-to the jury that "hackers" are, or can be, seedy lowlife characters,
-but it may have damaged Prophet's credibility somewhat.
-Zenner later suggested that drugs might have damaged Prophet's memory.
-The interesting fact also surfaced that Prophet had never
-physically met Craig Neidorf. He didn't even know
-Neidorf's last name--at least, not until the trial.
-
-Prophet confirmed the basic facts of his hacker career.
-He was a member of the Legion of Doom. He had abused codes,
-he had broken into switching stations and re-routed calls,
-he had hung out on pirate bulletin boards. He had raided
-the BellSouth AIMSX computer, copied the E911 Document,
-stored it on Jolnet, mailed it to Neidorf. He and Neidorf
-had edited it, and Neidorf had known where it came from.
-
-Zenner, however, had Prophet confirm that Neidorf was not a member
-of the Legion of Doom, and had not urged Prophet to break into
-BellSouth computers. Neidorf had never urged Prophet to defraud anyone,
-or to steal anything. Prophet also admitted that he had never known Neidorf
-to break in to any computer. Prophet said that no one in the Legion of Doom
-considered Craig Neidorf a "hacker" at all. Neidorf was not a UNIX maven,
-and simply lacked the necessary skill and ability to break into computers.
-Neidorf just published a magazine.
-
-On Friday, July 27, 1990, the case against Neidorf collapsed.
-Cook moved to dismiss the indictment, citing "information currently
-available to us that was not available to us at the inception of the trial."
-Judge Bua praised the prosecution for this action, which he described as
-"very responsible," then dismissed a juror and declared a mistrial.
-
-Neidorf was a free man. His defense, however, had cost himself
-and his family dearly. Months of his life had been consumed in anguish;
-he had seen his closest friends shun him as a federal criminal.
-He owed his lawyers over a hundred thousand dollars, despite
-a generous payment to the defense by Mitch Kapor.
-
-Neidorf was not found innocent. The trial was simply dropped.
-Nevertheless, on September 9, 1991, Judge Bua granted Neidorf's
-motion for the "expungement and sealing" of his indictment record.
-The United States Secret Service was ordered to delete and destroy
-all fingerprints, photographs, and other records of arrest
-or processing relating to Neidorf's indictment, including
-their paper documents and their computer records.
-
-Neidorf went back to school, blazingly determined to become a lawyer.
-Having seen the justice system at work, Neidorf lost much of his enthusiasm
-for merely technical power. At this writing, Craig Neidorf is working
-in Washington as a salaried researcher for the American Civil Liberties Union.
-
-#
-
-The outcome of the Neidorf trial changed the EFF
-from voices-in-the-wilderness to the media darlings
-of the new frontier.
-
-Legally speaking, the Neidorf case was not a sweeping triumph
-for anyone concerned. No constitutional principles had been established.
-The issues of "freedom of the press" for electronic publishers remained
-in legal limbo. There were public misconceptions about the case.
-Many people thought Neidorf had been found innocent and relieved
-of all his legal debts by Kapor. The truth was that the government
-had simply dropped the case, and Neidorf's family had gone deeply
-into hock to support him.
-
-But the Neidorf case did provide a single, devastating, public sound-bite:
-THE FEDS SAID IT WAS WORTH EIGHTY GRAND, AND IT WAS ONLY WORTH THIRTEEN BUCKS.
-
-This is the Neidorf case's single most memorable element. No serious report
-of the case missed this particular element. Even cops could not read this
-without a wince and a shake of the head. It left the public credibility
-of the crackdown agents in tatters.
-
-The crackdown, in fact, continued, however. Those two charges
-against Prophet, which had been based on the E911 Document,
-were quietly forgotten at his sentencing--even though Prophet
-had already pled guilty to them. Georgia federal prosecutors
-strongly argued for jail time for the Atlanta Three, insisting on
-"the need to send a message to the community," "the message that
-hackers around the country need to hear."
-
-There was a great deal in their sentencing memorandum
-about the awful things that various other hackers had done
-(though the Atlanta Three themselves had not, in fact,
-actually committed these crimes). There was also much
-speculation about the awful things that the Atlanta Three
-MIGHT have done and WERE CAPABLE of doing (even though
-they had not, in fact, actually done them).
-The prosecution's argument carried the day.
-The Atlanta Three were sent to prison:
-Urvile and Leftist both got 14 months each,
-while Prophet (a second offender) got 21 months.
-
-The Atlanta Three were also assessed staggering fines as "restitution":
-$233,000 each. BellSouth claimed that the defendants had "stolen"
-"approximately $233,880 worth" of "proprietary computer access information"--
-specifically, $233,880 worth of computer passwords and connect addresses.
-BellSouth's astonishing claim of the extreme value of its own computer
-passwords and addresses was accepted at face value by the Georgia court.
-Furthermore (as if to emphasize its theoretical nature) this enormous sum
-was not divvied up among the Atlanta Three, but each of them had to pay
-all of it.
-
-A striking aspect of the sentence was that the Atlanta Three were
-specifically forbidden to use computers, except for work or under supervision.
-Depriving hackers of home computers and modems makes some sense if one
-considers hackers as "computer addicts," but EFF, filing an amicus brief
-in the case, protested that this punishment was unconstitutional--
-it deprived the Atlanta Three of their rights of free association
-and free expression through electronic media.
-
-Terminus, the "ultimate hacker," was finally sent to prison for a year
-through the dogged efforts of the Chicago Task Force. His crime,
-to which he pled guilty, was the transfer of the UNIX password trapper,
-which was officially valued by AT&T at $77,000, a figure which aroused
-intense skepticism among those familiar with UNIX "login.c" programs.
-
-The jailing of Terminus and the Atlanta Legionnaires of Doom, however,
-did not cause the EFF any sense of embarrassment or defeat.
-On the contrary, the civil libertarians were rapidly gathering strength.
-
-An early and potent supporter was Senator Patrick Leahy,
-Democrat from Vermont, who had been a Senate sponsor
-of the Electronic Communications Privacy Act. Even before
-the Neidorf trial, Leahy had spoken out in defense of hacker-power
-and freedom of the keyboard: "We cannot unduly inhibit the inquisitive
-13-year-old who, if left to experiment today, may tomorrow develop
-the telecommunications or computer technology to lead the United States
-into the 21st century. He represents our future and our best hope
-to remain a technologically competitive nation."
-
-It was a handsome statement, rendered perhaps rather more effective
-by the fact that the crackdown raiders DID NOT HAVE any Senators
-speaking out for THEM. On the contrary, their highly secretive
-actions and tactics, all "sealed search warrants" here and
-"confidential ongoing investigations" there, might have won
-them a burst of glamorous publicity at first, but were crippling
-them in the on-going propaganda war. Gail Thackeray was reduced
-to unsupported bluster: "Some of these people who are loudest
-on the bandwagon may just slink into the background,"
-she predicted in Newsweek--when all the facts came out,
-and the cops were vindicated.
-
-But all the facts did not come out. Those facts that did,
-were not very flattering. And the cops were not vindicated.
-And Gail Thackeray lost her job. By the end of 1991,
-William Cook had also left public employment.
-
-1990 had belonged to the crackdown, but by '91 its agents
-were in severe disarray, and the libertarians were on a roll.
-People were flocking to the cause.
-
-A particularly interesting ally had been Mike Godwin of Austin, Texas.
-Godwin was an individual almost as difficult to describe as Barlow;
-he had been editor of the student newspaper of the University of Texas,
-and a computer salesman, and a programmer, and in 1990 was back
-in law school, looking for a law degree.
-
-Godwin was also a bulletin board maven. He was very well-known
-in the Austin board community under his handle "Johnny Mnemonic,"
-which he adopted from a cyberpunk science fiction story by William Gibson.
-Godwin was an ardent cyberpunk science fiction fan. As a fellow Austinite
-of similar age and similar interests, I myself had known Godwin socially
-for many years. When William Gibson and myself had been writing our
-collaborative SF novel, The Difference Engine, Godwin had been our
-technical advisor in our effort to link our Apple word-processors
-from Austin to Vancouver. Gibson and I were so pleased by his generous
-expert help that we named a character in the novel "Michael Godwin"
-in his honor.
-
-The handle "Mnemonic" suited Godwin very well. His erudition
-and his mastery of trivia were impressive to the point of stupor;
-his ardent curiosity seemed insatiable, and his desire to debate
-and argue seemed the central drive of his life. Godwin had even
-started his own Austin debating society, wryly known as the
-"Dull Men's Club." In person, Godwin could be overwhelming;
-a flypaper-brained polymath who could not seem to let any idea go.
-On bulletin boards, however, Godwin's closely reasoned,
-highly grammatical, erudite posts suited the medium well,
-and he became a local board celebrity.
-
-Mike Godwin was the man most responsible for the public national exposure
-of the Steve Jackson case. The Izenberg seizure in Austin had received
-no press coverage at all. The March 1 raids on Mentor, Bloodaxe, and
-Steve Jackson Games had received a brief front-page splash in the
-front page of the Austin American-Statesman, but it was confused
-and ill-informed: the warrants were sealed, and the Secret Service
-wasn't talking. Steve Jackson seemed doomed to obscurity.
-Jackson had not been arrested; he was not charged with any crime;
-he was not on trial. He had lost some computers in an ongoing
-investigation--so what? Jackson tried hard to attract attention
-to the true extent of his plight, but he was drawing a blank;
-no one in a position to help him seemed able to get a mental grip
-on the issues.
-
-Godwin, however, was uniquely, almost magically, qualified
-to carry Jackson's case to the outside world. Godwin was
-a board enthusiast, a science fiction fan, a former journalist,
-a computer salesman, a lawyer-to-be, and an Austinite.
-Through a coincidence yet more amazing, in his last year
-of law school Godwin had specialized in federal prosecutions
-and criminal procedure. Acting entirely on his own, Godwin made
-up a press packet which summarized the issues and provided useful
-contacts for reporters. Godwin's behind-the-scenes effort
-(which he carried out mostly to prove a point in a local board debate)
-broke the story again in the Austin American-Statesman and then in Newsweek.
-
-Life was never the same for Mike Godwin after that. As he joined the growing
-civil liberties debate on the Internet, it was obvious to all parties involved
-that here was one guy who, in the midst of complete murk and confusion,
-GENUINELY UNDERSTOOD EVERYTHING HE WAS TALKING ABOUT. The disparate elements
-of Godwin's dilettantish existence suddenly fell together as neatly as
-the facets of a Rubik's cube.
-
-When the time came to hire a full-time EFF staff attorney,
-Godwin was the obvious choice. He took the Texas bar exam,
-left Austin, moved to Cambridge, became a full-time, professional,
-computer civil libertarian, and was soon touring the nation on behalf
-of EFF, delivering well-received addresses on the issues to crowds
-as disparate as academics, industrialists, science fiction fans,
-and federal cops.
-
-Michael Godwin is currently the chief legal counsel of
-the Electronic Frontier Foundation in Cambridge, Massachusetts.
-
-#
-
-Another early and influential participant in the controversy
-was Dorothy Denning. Dr. Denning was unique among investigators
-of the computer underground in that she did not enter the debate
-with any set of politicized motives. She was a professional
-cryptographer and computer security expert whose primary interest
-in hackers was SCHOLARLY. She had a B.A. and M.A. in mathematics,
-and a Ph.D. in computer science from Purdue. She had worked for SRI
-International, the California think-tank that was also the home of
-computer-security maven Donn Parker, and had authored an influential text
-called Cryptography and Data Security. In 1990, Dr. Denning was working for
-Digital Equipment Corporation in their Systems Reseach Center. Her husband,
-Peter Denning, was also a computer security expert, working for NASA's
-Research Institute for Advanced Computer Science. He had edited the
-well-received Computers Under Attack: Intruders, Worms and Viruses.
-
-Dr. Denning took it upon herself to contact the digital underground,
-more or less with an anthropological interest. There she discovered
-that these computer-intruding hackers, who had been characterized
-as unethical, irresponsible, and a serious danger to society,
-did in fact have their own subculture and their own rules.
-They were not particularly well-considered rules, but they were,
-in fact, rules. Basically, they didn't take money and they
-didn't break anything.
-
-Her dispassionate reports on her researches did a great deal
-to influence serious-minded computer professionals--the sort
-of people who merely rolled their eyes at the cyberspace
-rhapsodies of a John Perry Barlow.
-
-For young hackers of the digital underground, meeting Dorothy Denning
-was a genuinely mind-boggling experience. Here was this neatly coiffed,
-conservatively dressed, dainty little personage, who reminded most
-hackers of their moms or their aunts. And yet she was an IBM systems
-programmer with profound expertise in computer architectures
-and high-security information flow, who had personal friends
-in the FBI and the National Security Agency.
-
-Dorothy Denning was a shining example of the American mathematical
-intelligentsia, a genuinely brilliant person from the central ranks
-of the computer-science elite. And here she was, gently questioning
-twenty-year-old hairy-eyed phone-phreaks over the deeper ethical
-implications of their behavior.
-
-Confronted by this genuinely nice lady, most hackers sat up very straight
-and did their best to keep the anarchy-file stuff down to a faint whiff
-of brimstone. Nevertheless, the hackers WERE in fact prepared to seriously
-discuss serious issues with Dorothy Denning. They were willing to speak
-the unspeakable and defend the indefensible, to blurt out their convictions
-that information cannot be owned, that the databases of governments and large
-corporations were a threat to the rights and privacy of individuals.
-
-Denning's articles made it clear to many that "hacking"
-was not simple vandalism by some evil clique of psychotics.
-"Hacking" was not an aberrant menace that could be charmed away
-by ignoring it, or swept out of existence by jailing a few ringleaders.
-Instead, "hacking" was symptomatic of a growing, primal struggle over
-knowledge and power in the age of information.
-
-Denning pointed out that the attitude of hackers were at least partially
-shared by forward-looking management theorists in the business community:
-people like Peter Drucker and Tom Peters. Peter Drucker, in his book
-The New Realities, had stated that "control of information by the government
-is no longer possible. Indeed, information is now transnational.
-Like money, it has no `fatherland.'"
-
-And management maven Tom Peters had chided large corporations for uptight,
-proprietary attitudes in his bestseller, Thriving on Chaos:
-"Information hoarding, especially by politically motivated,
-power-seeking staffs, had been commonplace throughout American industry,
-service and manufacturing alike. It will be an impossible
-millstone aroung the neck of tomorrow's organizations."
-
-Dorothy Denning had shattered the social membrane of the
-digital underground. She attended the Neidorf trial,
-where she was prepared to testify for the defense as an expert witness.
-She was a behind-the-scenes organizer of two of the most important
-national meetings of the computer civil libertarians. Though not
-a zealot of any description, she brought disparate elements of the
-electronic community into a surprising and fruitful collusion.
-
-Dorothy Denning is currently the Chair of the Computer Science Department
-at Georgetown University in Washington, DC.
-
-#
-
-There were many stellar figures in the civil libertarian community.
-There's no question, however, that its single most influential figure
-was Mitchell D. Kapor. Other people might have formal titles,
-or governmental positions, have more experience with crime,
-or with the law, or with the arcanities of computer security
-or constitutional theory. But by 1991 Kapor had transcended
-any such narrow role. Kapor had become "Mitch."
-
-Mitch had become the central civil-libertarian ad-hocrat.
-Mitch had stood up first, he had spoken out loudly, directly,
-vigorously and angrily, he had put his own reputation,
-and his very considerable personal fortune, on the line.
-By mid-'91 Kapor was the best-known advocate of his cause
-and was known PERSONALLY by almost every single human being in America
-with any direct influence on the question of civil liberties in cyberspace.
-Mitch had built bridges, crossed voids, changed paradigms, forged metaphors,
-made phone-calls and swapped business cards to such spectacular effect
-that it had become impossible for anyone to take any action in the
-"hacker question" without wondering what Mitch might think--
-and say--and tell his friends.
-
-The EFF had simply NETWORKED the situation into an entirely new status quo.
-And in fact this had been EFF's deliberate strategy from the beginning.
-Both Barlow and Kapor loathed bureaucracies and had deliberately
-chosen to work almost entirely through the electronic spiderweb of
-"valuable personal contacts."
-
-After a year of EFF, both Barlow and Kapor had every reason
-to look back with satisfaction. EFF had established its own Internet node,
-"eff.org," with a well-stocked electronic archive of documents on
-electronic civil rights, privacy issues, and academic freedom.
-EFF was also publishing EFFector, a quarterly printed journal,
-as well as EFFector Online, an electronic newsletter with
-over 1,200 subscribers. And EFF was thriving on the Well.
-
-EFF had a national headquarters in Cambridge and a full-time staff.
-It had become a membership organization and was attracting
-grass-roots support. It had also attracted the support
-of some thirty civil-rights lawyers, ready and eager
-to do pro bono work in defense of the Constitution in Cyberspace.
-
-EFF had lobbied successfully in Washington and in Massachusetts
-to change state and federal legislation on computer networking.
-Kapor in particular had become a veteran expert witness,
-and had joined the Computer Science and Telecommunications Board
-of the National Academy of Science and Engineering.
-
-EFF had sponsored meetings such as "Computers, Freedom and Privacy"
-and the CPSR Roundtable. It had carried out a press offensive that,
-in the words of EFFector, "has affected the climate of opinion about
-computer networking and begun to reverse the slide into
-`hacker hysteria' that was beginning to grip the nation."
-
-It had helped Craig Neidorf avoid prison.
-
-And, last but certainly not least, the Electronic Frontier Foundation
-had filed a federal lawsuit in the name of Steve Jackson,
-Steve Jackson Games Inc., and three users of the Illuminati
-bulletin board system. The defendants were, and are,
-the United States Secret Service, William Cook, Tim Foley,
-Barbara Golden and Henry Kleupfel.
-
-The case, which is in pre-trial procedures in an Austin federal court
-as of this writing, is a civil action for damages to redress
-alleged violations of the First and Fourth Amendments to the
-United States Constitution, as well as the Privacy Protection Act
-of 1980 (42 USC 2000aa et seq.), and the Electronic Communications
-Privacy Act (18 USC 2510 et seq and 2701 et seq).
-
-EFF had established that it had credibility. It had also established
-that it had teeth.
-
-In the fall of 1991 I travelled to Massachusetts to speak personally
-with Mitch Kapor. It was my final interview for this book.
-
-#
-
-The city of Boston has always been one of the major intellectual centers
-of the American republic. It is a very old city by American standards,
-a place of skyscrapers overshadowing seventeenth-century graveyards,
-where the high-tech start-up companies of Route 128 co-exist with the
-hand-wrought pre-industrial grace of "Old Ironsides," the USS CONSTITUTION.
-
-The Battle of Bunker Hill, one of the first and bitterest armed clashes
-of the American Revolution, was fought in Boston's environs. Today there is
-a monumental spire on Bunker Hill, visible throughout much of the city.
-The willingness of the republican revolutionaries to take up arms and fire
-on their oppressors has left a cultural legacy that two full centuries
-have not effaced. Bunker Hill is still a potent center of American political
-symbolism, and the Spirit of '76 is still a potent image for those who seek
-to mold public opinion.
-
-Of course, not everyone who wraps himself in the flag is necessarily
-a patriot. When I visited the spire in September 1991, it bore a huge,
-badly-erased, spray-can grafitto around its bottom reading
-"BRITS OUT--IRA PROVOS." Inside this hallowed edifice was
-a glass-cased diorama of thousands of tiny toy soldiers,
-rebels and redcoats, fighting and dying over the green hill,
-the riverside marshes, the rebel trenchworks. Plaques indicated the
-movement of troops, the shiftings of strategy. The Bunker Hill Monument
-is occupied at its very center by the toy soldiers of a military
-war-game simulation.
-
-The Boston metroplex is a place of great universities,
-prominent among the Massachusetts Institute of Technology,
-where the term "computer hacker" was first coined. The Hacker Crackdown
-of 1990 might be interpreted as a political struggle among American cities:
-traditional strongholds of longhair intellectual liberalism,
-such as Boston, San Francisco, and Austin, versus the bare-knuckle
-industrial pragmatism of Chicago and Phoenix (with Atlanta and New York
-wrapped in internal struggle).
-
-The headquarters of the Electronic Frontier Foundation is on
-155 Second Street in Cambridge, a Bostonian suburb north
-of the River Charles. Second Street has weedy sidewalks of dented,
-sagging brick and elderly cracked asphalt; large street-signs warn
-"NO PARKING DURING DECLARED SNOW EMERGENCY." This is an old area
-of modest manufacturing industries; the EFF is catecorner from the
-Greene Rubber Company. EFF's building is two stories of red brick;
-its large wooden windows feature gracefully arched tops and stone sills.
-
-The glass window beside the Second Street entrance bears three sheets
-of neatly laser-printed paper, taped against the glass. They read:
-ON Technology. EFF. KEI.
-
-"ON Technology" is Kapor's software company, which currently specializes
-in "groupware" for the Apple Macintosh computer. "Groupware" is intended
-to promote efficient social interaction among office-workers linked
-by computers. ON Technology's most successful software products to date
-are "Meeting Maker" and "Instant Update."
-
-"KEI" is Kapor Enterprises Inc., Kapor's personal holding company,
-the commercial entity that formally controls his extensive investments
-in other hardware and software corporations.
-
-"EFF" is a political action group--of a special sort.
-
-Inside, someone's bike has been chained to the handrails
-of a modest flight of stairs. A wall of modish glass brick
-separates this anteroom from the offices. Beyond the brick,
-there's an alarm system mounted on the wall, a sleek, complex little
-number that resembles a cross between a thermostat and a CD player.
-Piled against the wall are box after box of a recent special issue
-of Scientific American, "How to Work, Play, and Thrive in Cyberspace,"
-with extensive coverage of electronic networking techniques
-and political issues, including an article by Kapor himself.
-These boxes are addressed to Gerard Van der Leun, EFF's
-Director of Communications, who will shortly mail those magazines
-to every member of the EFF.
-
-The joint headquarters of EFF, KEI, and ON Technology,
-which Kapor currently rents, is a modestly bustling place.
-It's very much the same physical size as Steve Jackson's gaming company.
-It's certainly a far cry from the gigantic gray steel-sided railway
-shipping barn, on the Monsignor O'Brien Highway, that is owned
-by Lotus Development Corporation.
-
-Lotus is, of course, the software giant that Mitchell Kapor founded
-in the late 70s. The software program Kapor co-authored,
-"Lotus 1-2-3," is still that company's most profitable product.
-"Lotus 1-2-3" also bears a singular distinction in the
-digital underground: it's probably the most pirated piece
-of application software in world history.
-
-Kapor greets me cordially in his own office, down a hall.
-Kapor, whose name is pronounced KAY-por, is in his early forties,
-married and the father of two. He has a round face, high forehead,
-straight nose, a slightly tousled mop of black hair peppered with gray.
-His large brown eyes are wideset, reflective, one might almost say soulful.
-He disdains ties, and commonly wears Hawaiian shirts and tropical prints,
-not so much garish as simply cheerful and just that little bit anomalous.
-
-There is just the whiff of hacker brimstone about Mitch Kapor.
-He may not have the hard-riding, hell-for-leather, guitar-strumming
-charisma of his Wyoming colleague John Perry Barlow, but there's
-something about the guy that still stops one short. He has the air
-of the Eastern city dude in the bowler hat, the dreamy,
-Longfellow-quoting poker shark who only HAPPENS to know
-the exact mathematical odds against drawing to an inside straight.
-Even among his computer-community colleagues, who are hardly known
-for mental sluggishness, Kapor strikes one forcefully as a very
-intelligent man. He speaks rapidly, with vigorous gestures,
-his Boston accent sometimes slipping to the sharp nasal tang
-of his youth in Long Island.
-
-Kapor, whose Kapor Family Foundation does much of his philanthropic work,
-is a strong supporter of Boston's Computer Museum. Kapor's interest
-in the history of his industry has brought him some remarkable curios,
-such as the "byte" just outside his office door. This "byte"--
-eight digital bits--has been salvaged from the wreck of an
-electronic computer of the pre-transistor age. It's a standing gunmetal
-rack about the size of a small toaster-oven: with eight slots
-of hand-soldered breadboarding featuring thumb-sized vacuum tubes.
-If it fell off a table it could easily break your foot,
-but it was state-of-the-art computation in the 1940s.
-(It would take exactly 157,184 of these primordial toasters
-to hold the first part of this book.)
-
-There's also a coiling, multicolored, scaly dragon that some
-inspired techno-punk artist has cobbled up entirely out of transistors,
-capacitors, and brightly plastic-coated wiring.
-
-Inside the office, Kapor excuses himself briefly to do a little
-mouse-whizzing housekeeping on his personal Macintosh IIfx.
-If its giant screen were an open window, an agile person
-could climb through it without much trouble at all.
-There's a coffee-cup at Kapor's elbow, a memento of his
-recent trip to Eastern Europe, which has a black-and-white
-stencilled photo and the legend CAPITALIST FOOLS TOUR.
-It's Kapor, Barlow, and two California venture-capitalist luminaries
-of their acquaintance, four windblown, grinning Baby Boomer
-dudes in leather jackets, boots, denim, travel bags,
-standing on airport tarmac somewhere behind the formerly Iron Curtain.
-They look as if they're having the absolute time of their lives.
-
-Kapor is in a reminiscent mood. We talk a bit about his youth--
-high school days as a "math nerd," Saturdays attending Columbia University's
-high-school science honors program, where he had his first experience
-programming computers. IBM 1620s, in 1965 and '66. "I was very interested,"
-says Kapor, "and then I went off to college and got distracted by drugs sex
-and rock and roll, like anybody with half a brain would have then!"
-After college he was a progressive-rock DJ in Hartford, Connecticut,
-for a couple of years.
-
-I ask him if he ever misses his rock and roll days--if he ever wished
-he could go back to radio work.
-
-He shakes his head flatly. "I stopped thinking about going back
-to be a DJ the day after Altamont."
-
-Kapor moved to Boston in 1974 and got a job programming mainframes in COBOL.
-He hated it. He quit and became a teacher of transcendental meditation.
-(It was Kapor's long flirtation with Eastern mysticism that gave the
-world "Lotus.")
-
-In 1976 Kapor went to Switzerland, where the Transcendental Meditation
-movement had rented a gigantic Victorian hotel in St-Moritz. It was
-an all-male group--a hundred and twenty of them--determined upon
-Enlightenment or Bust. Kapor had given the transcendant his best shot.
-He was becoming disenchanted by "the nuttiness in the organization."
-"They were teaching people to levitate," he says, staring at the floor.
-His voice drops an octave, becomes flat. "THEY DON'T LEVITATE."
-
-Kapor chose Bust. He went back to the States and acquired a degree
-in counselling psychology. He worked a while in a hospital,
-couldn't stand that either. "My rep was," he says "a very bright kid
-with a lot of potential who hasn't found himself. Almost thirty.
-Sort of lost."
-
-Kapor was unemployed when he bought his first personal computer--an Apple II.
-He sold his stereo to raise cash and drove to New Hampshire to avoid the
-sales tax.
-
-"The day after I purchased it," Kapor tells me, "I was hanging out
-in a computer store and I saw another guy, a man in his forties,
-well-dressed guy, and eavesdropped on his conversation with the salesman.
-He didn't know anything about computers. I'd had a year programming.
-And I could program in BASIC. I'd taught myself. So I went up to him,
-and I actually sold myself to him as a consultant." He pauses.
-"I don't know where I got the nerve to do this. It was uncharacteristic.
-I just said, `I think I can help you, I've been listening,
-this is what you need to do and I think I can do it for you.'
-And he took me on! He was my first client! I became a computer
-consultant the first day after I bought the Apple II."
-
-Kapor had found his true vocation. He attracted more clients
-for his consultant service, and started an Apple users' group.
-
-A friend of Kapor's, Eric Rosenfeld, a graduate student at MIT,
-had a problem. He was doing a thesis on an arcane form of
-financial statistics, but could not wedge himself into the crowded queue
-for time on MIT's mainframes. (One might note at this point that if
-Mr. Rosenfeld had dishonestly broken into the MIT mainframes,
-Kapor himself might have never invented Lotus 1-2-3 and
-the PC business might have been set back for years!)
-Eric Rosenfeld did have an Apple II, however,
-and he thought it might be possible to scale the problem down.
-Kapor, as favor, wrote a program for him in BASIC that did the job.
-
-It then occurred to the two of them, out of the blue,
-that it might be possible to SELL this program.
-They marketed it themselves, in plastic baggies,
-for about a hundred bucks a pop, mail order.
-"This was a total cottage industry by a marginal consultant,"
-Kapor says proudly. "That's how I got started, honest to God."
-
-Rosenfeld, who later became a very prominent figure on Wall Street,
-urged Kapor to go to MIT's business school for an MBA.
-Kapor did seven months there, but never got his MBA.
-He picked up some useful tools--mainly a firm grasp
-of the principles of accounting--and, in his own words,
-"learned to talk MBA." Then he dropped out and went to Silicon Valley.
-
-The inventors of VisiCalc, the Apple computer's premier business program,
-had shown an interest in Mitch Kapor. Kapor worked diligently for them
-for six months, got tired of California, and went back to Boston
-where they had better bookstores. The VisiCalc group had made
-the critical error of bringing in "professional management."
-"That drove them into the ground," Kapor says.
-
-"Yeah, you don't hear a lot about VisiCalc these days," I muse.
-
-Kapor looks surprised. "Well, Lotus. . . we BOUGHT it."
-
-"Oh. You BOUGHT it?"
-
-"Yeah."
-
-"Sort of like the Bell System buying Western Union?"
-
-Kapor grins. "Yep! Yep! Yeah, exactly!"
-
-Mitch Kapor was not in full command of the destiny of himself
-or his industry. The hottest software commodities of the early 1980s
-were COMPUTER GAMES--the Atari seemed destined to enter every teenage home
-in America. Kapor got into business software simply because he didn't have
-any particular feeling for computer games. But he was supremely fast
-on his feet, open to new ideas and inclined to trust his instincts.
-And his instincts were good. He chose good people to deal with--
-gifted programmer Jonathan Sachs (the co-author of Lotus 1-2-3).
-Financial wizard Eric Rosenfeld, canny Wall Street analyst
-and venture capitalist Ben Rosen. Kapor was the founder and CEO of Lotus,
-one of the most spectacularly successful business ventures of the
-later twentieth century.
-
-He is now an extremely wealthy man. I ask him if he actually
-knows how much money he has.
-
-"Yeah," he says. "Within a percent or two."
-
-How much does he actually have, then?
-
-He shakes his head. "A lot. A lot. Not something I talk about.
-Issues of money and class are things that cut pretty close to the bone."
-
-I don't pry. It's beside the point. One might presume, impolitely,
-that Kapor has at least forty million--that's what he got the year
-he left Lotus. People who ought to know claim Kapor has about
-a hundred and fifty million, give or take a market swing
-in his stock holdings. If Kapor had stuck with Lotus,
-as his colleague friend and rival Bill Gates has stuck
-with his own software start-up, Microsoft, then Kapor
-would likely have much the same fortune Gates has--
-somewhere in the neighborhood of three billion,
-give or take a few hundred million. Mitch Kapor
-has all the money he wants. Money has lost whatever charm
-it ever held for him--probably not much in the first place.
-When Lotus became too uptight, too bureaucratic, too far
-from the true sources of his own satisfaction, Kapor walked.
-He simply severed all connections with the company and went out the door.
-It stunned everyone--except those who knew him best.
-
-Kapor has not had to strain his resources to wreak a thorough
-transformation in cyberspace politics. In its first year,
-EFF's budget was about a quarter of a million dollars.
-Kapor is running EFF out of his pocket change.
-
-Kapor takes pains to tell me that he does not consider himself
-a civil libertarian per se. He has spent quite some time
-with true-blue civil libertarians lately, and there's a
-political-correctness to them that bugs him. They seem
-to him to spend entirely too much time in legal nitpicking
-and not enough vigorously exercising civil rights in the
-everyday real world.
-
-Kapor is an entrepreneur. Like all hackers, he prefers his involvements
-direct, personal, and hands-on. "The fact that EFF has a node on the
-Internet is a great thing. We're a publisher. We're a distributor
-of information." Among the items the eff.org Internet node carries
-is back issues of Phrack. They had an internal debate about that in EFF,
-and finally decided to take the plunge. They might carry other
-digital underground publications--but if they do, he says,
-"we'll certainly carry Donn Parker, and anything Gail Thackeray
-wants to put up. We'll turn it into a public library, that has
-the whole spectrum of use. Evolve in the direction of people making up
-their own minds." He grins. "We'll try to label all the editorials."
-
-Kapor is determined to tackle the technicalities of the Internet
-in the service of the public interest. "The problem with being a node
-on the Net today is that you've got to have a captive technical specialist.
-We have Chris Davis around, for the care and feeding of the balky beast!
-We couldn't do it ourselves!"
-
-He pauses. "So one direction in which technology has to evolve
-is much more standardized units, that a non-technical person
-can feel comfortable with. It's the same shift as from minicomputers to PCs.
-I can see a future in which any person can have a Node on the Net.
-Any person can be a publisher. It's better than the media we now have.
-It's possible. We're working actively."
-
-Kapor is in his element now, fluent, thoroughly in command in his material.
-"You go tell a hardware Internet hacker that everyone should have a node
-on the Net," he says, "and the first thing they're going to say is,
-`IP doesn't scale!'" ("IP" is the interface protocol for the Internet.
-As it currently exists, the IP software is simply not capable of
-indefinite expansion; it will run out of usable addresses, it will saturate.)
-"The answer," Kapor says, "is: evolve the protocol! Get the smart people
-together and figure out what to do. Do we add ID? Do we add new protocol?
-Don't just say, WE CAN'T DO IT."
-
-Getting smart people together to figure out what to do is a skill
-at which Kapor clearly excels. I counter that people on the Internet
-rather enjoy their elite technical status, and don't seem particularly
-anxious to democratize the Net.
-
-Kapor agrees, with a show of scorn. "I tell them that this is the snobbery
-of the people on the Mayflower looking down their noses at the people
-who came over ON THE SECOND BOAT! Just because they got here a year,
-or five years, or ten years before everybody else, that doesn't give
-them ownership of cyberspace! By what right?"
-
-I remark that the telcos are an electronic network, too,
-and they seem to guard their specialized knowledge pretty closely.
-
-Kapor ripostes that the telcos and the Internet are entirely
-different animals. "The Internet is an open system,
-everything is published, everything gets argued about,
-basically by anybody who can get in. Mostly, it's exclusive
-and elitist just because it's so difficult. Let's make it easier to use."
-
-On the other hand, he allows with a swift change of emphasis,
-the so-called elitists do have a point as well. "Before people start coming in,
-who are new, who want to make suggestions, and criticize the Net as
-`all screwed up'. . . . They should at least take the time to understand
-the culture on its own terms. It has its own history--show some respect
-for it. I'm a conservative, to that extent."
-
-The Internet is Kapor's paradigm for the future of telecommunications.
-The Internet is decentralized, non-hierarchical, almost anarchic.
-There are no bosses, no chain of command, no secret data.
-If each node obeys the general interface standards,
-there's simply no need for any central network authority.
-
-Wouldn't that spell the doom of AT&T as an institution? I ask.
-
-That prospect doesn't faze Kapor for a moment. "Their big advantage,
-that they have now, is that they have all of the wiring.
-But two things are happening. Anyone with right-of-way
-is putting down fiber--Southern Pacific Railroad,
-people like that--there's enormous `dark fiber' laid in."
-("Dark Fiber" is fiber-optic cable, whose enormous capacity
-so exceeds the demands of current usage that much of the
-fiber still has no light-signals on it--it's still `dark,'
-awaiting future use.)
-
-"The other thing that's happening is the local-loop stuff
-is going to go wireless. Everyone from Bellcore to the cable TV
-companies to AT&T wants to put in these things called
-`personal communication systems.' So you could have local competition--
-you could have multiplicity of people, a bunch of neighborhoods,
-sticking stuff up on poles. And a bunch of other people laying in dark fiber.
-So what happens to the telephone companies? There's enormous pressure
-on them from both sides.
-
-"The more I look at this, the more I believe that in a post-industrial,
-digital world, the idea of regulated monopolies is bad. People will
-look back on it and say that in the 19th and 20th centuries
-the idea of public utilities was an okay compromise.
-You needed one set of wires in the ground. It was too economically
-inefficient, otherwise. And that meant one entity running it.
-But now, with pieces being wireless--the connections are going
-to be via high-level interfaces, not via wires. I mean, ULTIMATELY
-there are going to be wires--but the wires are just a commodity.
-Fiber, wireless. You no longer NEED a utility."
-
-Water utilities? Gas utilities?
-
-Of course we still need those, he agrees. "But when what you're moving
-is information, instead of physical substances, then you can play by
-a different set of rules. We're evolving those rules now!
-Hopefully you can have a much more decentralized system,
-and one in which there's more competition in the marketplace.
-
-"The role of government will be to make sure that nobody cheats.
-The proverbial `level playing field.' A policy that prevents monopolization.
-It should result in better service, lower prices, more choices,
-and local empowerment." He smiles. "I'm very big on local empowerment."
-
-Kapor is a man with a vision. It's a very novel vision which he
-and his allies are working out in considerable detail and with great energy.
-Dark, cynical, morbid cyberpunk that I am, I cannot avoid considering
-some of the darker implications of "decentralized, nonhierarchical,
-locally empowered" networking.
-
-I remark that some pundits have suggested that electronic networking--faxes,
-phones, small-scale photocopiers--played a strong role in dissolving
-the power of centralized communism and causing the collapse of the Warsaw Pact.
-
-Socialism is totally discredited, says Kapor, fresh back from
-the Eastern Bloc. The idea that faxes did it, all by themselves,
-is rather wishful thinking.
-
-Has it occurred to him that electronic networking might corrode
-America's industrial and political infrastructure to the point
-where the whole thing becomes untenable, unworkable--and the old order
-just collapses headlong, like in Eastern Europe?
-
-"No," Kapor says flatly. "I think that's extraordinarily unlikely.
-In part, because ten or fifteen years ago, I had similar hopes
-about personal computers--which utterly failed to materialize."
-He grins wryly, then his eyes narrow. "I'm VERY opposed to techno-utopias.
-Every time I see one, I either run away, or try to kill it."
-
-It dawns on me then that Mitch Kapor is not trying to
-make the world safe for democracy. He certainly is not
-trying to make it safe for anarchists or utopians--
-least of all for computer intruders or electronic rip-off artists.
-What he really hopes to do is make the world safe for
-future Mitch Kapors. This world of decentralized, small-scale nodes,
-with instant global access for the best and brightest,
-would be a perfect milieu for the shoestring attic capitalism
-that made Mitch Kapor what he is today.
-
-Kapor is a very bright man. He has a rare combination
-of visionary intensity with a strong practical streak.
-The Board of the EFF: John Barlow, Jerry Berman of the ACLU,
-Stewart Brand, John Gilmore, Steve Wozniak, and Esther Dyson,
-the doyenne of East-West computer entrepreneurism--share his gift,
-his vision, and his formidable networking talents.
-They are people of the 1960s, winnowed-out by its turbulence
-and rewarded with wealth and influence. They are some of the best
-and the brightest that the electronic community has to offer.
-But can they do it, in the real world? Or are they only dreaming?
-They are so few. And there is so much against them.
-
-I leave Kapor and his networking employees struggling cheerfully
-with the promising intricacies of their newly installed Macintosh
-System 7 software. The next day is Saturday. EFF is closed.
-I pay a few visits to points of interest downtown.
-
-One of them is the birthplace of the telephone.
-
-It's marked by a bronze plaque in a plinth of black-and-white speckled granite. It sits in the
-plaza of the John F. Kennedy Federal Building, the very place where Kapor was
-once fingerprinted by the FBI.
-
-The plaque has a bas-relief picture of Bell's original telephone.
-"BIRTHPLACE OF THE TELEPHONE," it reads. "Here, on June 2, 1875,
-Alexander Graham Bell and Thomas A. Watson first transmitted sound over wires.
-
-"This successful experiment was completed in a fifth floor garret
-at what was then 109 Court Street and marked the beginning of
-world-wide telephone service."
-
-109 Court Street is long gone. Within sight of Bell's plaque,
-across a street, is one of the central offices of NYNEX,
-the local Bell RBOC, on 6 Bowdoin Square.
-
-I cross the street and circle the telco building, slowly,
-hands in my jacket pockets. It's a bright, windy, New England
-autumn day. The central office is a handsome 1940s-era megalith
-in late Art Deco, eight stories high.
-
-Parked outside the back is a power-generation truck.
-The generator strikes me as rather anomalous. Don't they
-already have their own generators in this eight-story monster?
-Then the suspicion strikes me that NYNEX must have heard
-of the September 17 AT&T power-outage which crashed New York City.
-Belt-and-suspenders, this generator. Very telco.
-
-Over the glass doors of the front entrance is a handsome bronze
-bas-relief of Art Deco vines, sunflowers, and birds, entwining
-the Bell logo and the legend NEW ENGLAND TELEPHONE AND TELEGRAPH COMPANY
---an entity which no longer officially exists.
-
-The doors are locked securely. I peer through the shadowed glass.
-Inside is an official poster reading:
-
-
-"New England Telephone a NYNEX Company
-
-ATTENTION
-
-"All persons while on New England Telephone
-Company premises are required to visibly wear their
-identification cards (C.C.P. Section 2, Page 1).
-
-"Visitors, vendors, contractors, and all others are
-required to visibly wear a daily pass.
-
-"Thank you.
-
-Kevin C. Stanton.
-Building Security Coordinator."
-
-
-Outside, around the corner, is a pull-down ribbed metal security door,
-a locked delivery entrance. Some passing stranger has grafitti-tagged
-this door, with a single word in red spray-painted cursive:
-
-Fury
-
-#
-
-My book on the Hacker Crackdown is almost over now.
-I have deliberately saved the best for last.
-
-In February 1991, I attended the CPSR Public Policy Roundtable,
-in Washington, DC. CPSR, Computer Professionals for Social Responsibility,
-was a sister organization of EFF, or perhaps its aunt, being older
-and perhaps somewhat wiser in the ways of the world of politics.
-
-Computer Professionals for Social Responsibility began in 1981
-in Palo Alto, as an informal discussion group of Californian
-computer scientists and technicians, united by nothing more
-than an electronic mailing list. This typical high-tech
-ad-hocracy received the dignity of its own acronym in 1982,
-and was formally incorporated in 1983.
-
-CPSR lobbied government and public alike with an educational
-outreach effort, sternly warning against any foolish
-and unthinking trust in complex computer systems.
-CPSR insisted that mere computers should never be
-considered a magic panacea for humanity's social,
-ethical or political problems. CPSR members were especially
-troubled about the stability, safety, and dependability
-of military computer systems, and very especially troubled
-by those systems controlling nuclear arsenals. CPSR was
-best-known for its persistent and well-publicized attacks on the
-scientific credibility of the Strategic Defense Initiative ("Star Wars").
-
-In 1990, CPSR was the nation's veteran cyber-political activist group,
-with over two thousand members in twenty- one local chapters across the US.
-It was especially active in Boston, Silicon Valley, and Washington DC,
-where its Washington office sponsored the Public Policy Roundtable.
-
-The Roundtable, however, had been funded by EFF, which had passed CPSR
-an extensive grant for operations. This was the first large-scale,
-official meeting of what was to become the electronic civil
-libertarian community.
-
-Sixty people attended, myself included--in this instance, not so much
-as a journalist as a cyberpunk author. Many of the luminaries
-of the field took part: Kapor and Godwin as a matter of course.
-Richard Civille and Marc Rotenberg of CPSR. Jerry Berman of the ACLU.
-John Quarterman, author of The Matrix. Steven Levy, author of Hackers.
-George Perry and Sandy Weiss of Prodigy Services, there to network
-about the civil-liberties troubles their young commercial
-network was experiencing. Dr. Dorothy Denning. Cliff Figallo,
-manager of the Well. Steve Jackson was there, having finally
-found his ideal target audience, and so was Craig Neidorf,
-"Knight Lightning" himself, with his attorney, Sheldon Zenner.
-Katie Hafner, science journalist, and co-author of Cyberpunk:
-Outlaws and Hackers on the Computer Frontier. Dave Farber,
-ARPAnet pioneer and fabled Internet guru. Janlori Goldman
-of the ACLU's Project on Privacy and Technology. John Nagle
-of Autodesk and the Well. Don Goldberg of the House Judiciary Committee.
-Tom Guidoboni, the defense attorney in the Internet Worm case.
-Lance Hoffman, computer-science professor at The George Washington
-University. Eli Noam of Columbia. And a host of others no less distinguished.
-
-Senator Patrick Leahy delivered the keynote address,
-expressing his determination to keep ahead of the curve
-on the issue of electronic free speech. The address was
-well-received, and the sense of excitement was palpable.
-Every panel discussion was interesting--some were entirely
-compelling. People networked with an almost frantic interest.
-
-I myself had a most interesting and cordial lunch discussion with
-Noel and Jeanne Gayler, Admiral Gayler being a former director
-of the National Security Agency. As this was the first known encounter
-between an actual no-kidding cyberpunk and a chief executive of
-America's largest and best-financed electronic espionage apparat,
-there was naturally a bit of eyebrow-raising on both sides.
-
-Unfortunately, our discussion was off-the-record. In fact
-all the discussions at the CPSR were officially off-the-record,
-the idea being to do some serious networking in an atmosphere
-of complete frankness, rather than to stage a media circus.
-
-In any case, CPSR Roundtable, though interesting and intensely valuable,
-was as nothing compared to the truly mind-boggling event that transpired
-a mere month later.
-
-#
-
-"Computers, Freedom and Privacy." Four hundred people from
-every conceivable corner of America's electronic community.
-As a science fiction writer, I have been to some weird gigs in my day,
-but this thing is truly BEYOND THE PALE. Even "Cyberthon,"
-Point Foundation's "Woodstock of Cyberspace" where Bay Area
-psychedelia collided headlong with the emergent world
-of computerized virtual reality, was like a Kiwanis Club gig
-compared to this astonishing do.
-
-The "electronic community" had reached an apogee.
-Almost every principal in this book is in attendance.
-Civil Libertarians. Computer Cops. The Digital Underground.
-Even a few discreet telco people. Colorcoded dots
-for lapel tags are distributed. Free Expression issues.
-Law Enforcement. Computer Security. Privacy. Journalists.
-Lawyers. Educators. Librarians. Programmers.
-Stylish punk-black dots for the hackers and phone phreaks.
-Almost everyone here seems to wear eight or nine dots,
-to have six or seven professional hats.
-
-It is a community. Something like Lebanon perhaps,
-but a digital nation. People who had feuded all year
-in the national press, people who entertained the deepest
-suspicions of one another's motives and ethics, are now
-in each others' laps. "Computers, Freedom and Privacy"
-had every reason in the world to turn ugly, and yet except
-for small irruptions of puzzling nonsense from the
-convention's token lunatic, a surprising bonhomie reigned.
-CFP was like a wedding-party in which two lovers,
-unstable bride and charlatan groom, tie the knot
-in a clearly disastrous matrimony.
-
-It is clear to both families--even to neighbors and random guests--
-that this is not a workable relationship, and yet the young couple's
-desperate attraction can brook no further delay. They simply cannot
-help themselves. Crockery will fly, shrieks from their newlywed home
-will wake the city block, divorce waits in the wings like a vulture
-over the Kalahari, and yet this is a wedding, and there is going
-to be a child from it. Tragedies end in death; comedies in marriage.
-The Hacker Crackdown is ending in marriage. And there will be a child.
-
-From the beginning, anomalies reign. John Perry Barlow,
-cyberspace ranger, is here. His color photo in
-The New York Times Magazine, Barlow scowling
-in a grim Wyoming snowscape, with long black coat,
-dark hat, a Macintosh SE30 propped on a fencepost
-and an awesome frontier rifle tucked under one arm,
-will be the single most striking visual image
-of the Hacker Crackdown. And he is CFP's guest of honor--
-along with Gail Thackeray of the FCIC! What on earth do
-they expect these dual guests to do with each other? Waltz?
-
-Barlow delivers the first address. Uncharacteristically,
-he is hoarse--the sheer volume of roadwork has worn him down.
-He speaks briefly, congenially, in a plea for conciliation,
-and takes his leave to a storm of applause.
-
-Then Gail Thackeray takes the stage. She's visibly nervous.
-She's been on the Well a lot lately. Reading those Barlow posts.
-Following Barlow is a challenge to anyone. In honor of the famous
-lyricist for the Grateful Dead, she announces reedily, she is going to read--
-A POEM. A poem she has composed herself.
-
-It's an awful poem, doggerel in the rollicking meter of Robert W. Service's
-The Cremation of Sam McGee, but it is in fact, a poem. It's the Ballad
-of the Electronic Frontier! A poem about the Hacker Crackdown and the
-sheer unlikelihood of CFP. It's full of in-jokes. The score or so cops
-in the audience, who are sitting together in a nervous claque,
-are absolutely cracking-up. Gail's poem is the funniest goddamn thing
-they've ever heard. The hackers and civil-libs, who had this woman figured
-for Ilsa She-Wolf of the SS, are staring with their jaws hanging loosely.
-Never in the wildest reaches of their imagination had they figured
-Gail Thackeray was capable of such a totally off-the-wall move.
-You can see them punching their mental CONTROL-RESET buttons.
-Jesus! This woman's a hacker weirdo! She's JUST LIKE US!
-God, this changes everything!
-
-Al Bayse, computer technician for the FBI, had been the only cop
-at the CPSR Roundtable, dragged there with his arm bent by
-Dorothy Denning. He was guarded and tightlipped at CPSR Roundtable;
-a "lion thrown to the Christians."
-
-At CFP, backed by a claque of cops, Bayse suddenly waxes eloquent
-and even droll, describing the FBI's "NCIC 2000", a gigantic digital catalog
-of criminal records, as if he has suddenly become some weird hybrid
-of George Orwell and George Gobel. Tentatively, he makes an arcane
-joke about statistical analysis. At least a third of the crowd laughs aloud.
-
-"They didn't laugh at that at my last speech," Bayse observes.
-He had been addressing cops--STRAIGHT cops, not computer people.
-It had been a worthy meeting, useful one supposes, but nothing like THIS.
-There has never been ANYTHING like this. Without any prodding,
-without any preparation, people in the audience simply begin to ask questions.
-Longhairs, freaky people, mathematicians. Bayse is answering, politely,
-frankly, fully, like a man walking on air. The ballroom's atmosphere
-crackles with surreality. A female lawyer behind me breaks into a sweat
-and a hot waft of surprisingly potent and musky perfume flows off
-her pulse-points.
-
-People are giddy with laughter. People are interested,
-fascinated, their eyes so wide and dark that they seem eroticized.
-Unlikely daisy-chains form in the halls, around the bar, on the escalators:
-cops with hackers, civil rights with FBI, Secret Service with phone phreaks.
-
-Gail Thackeray is at her crispest in a white wool sweater with a
-tiny Secret Service logo. "I found Phiber Optik at the payphones,
-and when he saw my sweater, he turned into a PILLAR OF SALT!" she chortles.
-
-Phiber discusses his case at much length with his arresting officer,
-Don Delaney of the New York State Police. After an hour's chat,
-the two of them look ready to begin singing "Auld Lang Syne."
-Phiber finally finds the courage to get his worst complaint off his chest.
-It isn't so much the arrest. It was the CHARGE. Pirating service
-off 900 numbers. I'm a PROGRAMMER, Phiber insists. This lame charge
-is going to hurt my reputation. It would have been cool to be busted
-for something happening, like Section 1030 computer intrusion.
-Maybe some kind of crime that's scarcely been invented yet.
-Not lousy phone fraud. Phooey.
-
-Delaney seems regretful. He had a mountain of possible criminal charges
-against Phiber Optik. The kid's gonna plead guilty anyway. He's a
-first timer, they always plead. Coulda charged the kid with most anything,
-and gotten the same result in the end. Delaney seems genuinely sorry
-not to have gratified Phiber in this harmless fashion. Too late now.
-Phiber's pled already. All water under the bridge. Whaddya gonna do?
-
-Delaney's got a good grasp on the hacker mentality.
-He held a press conference after he busted a bunch of
-Masters of Deception kids. Some journo had asked him:
-"Would you describe these people as GENIUSES?"
-Delaney's deadpan answer, perfect: "No, I would describe
-these people as DEFENDANTS." Delaney busts a kid for
-hacking codes with repeated random dialling. Tells the
-press that NYNEX can track this stuff in no time flat nowadays,
-and a kid has to be STUPID to do something so easy to catch.
-Dead on again: hackers don't mind being thought of as Genghis Khan
-by the straights, but if there's anything that really gets 'em
-where they live, it's being called DUMB.
-
-Won't be as much fun for Phiber next time around.
-As a second offender he's gonna see prison.
-Hackers break the law. They're not geniuses, either.
-They're gonna be defendants. And yet, Delaney muses over
-a drink in the hotel bar, he has found it impossible to treat
-them as common criminals. Delaney knows criminals. These kids,
-by comparison, are clueless--there is just no crook vibe off of them,
-they don't smell right, they're just not BAD.
-
-Delaney has seen a lot of action. He did Vietnam.
-He's been shot at, he has shot people. He's a homicide
-cop from New York. He has the appearance of a man who
-has not only seen the shit hit the fan but has seen it splattered
-across whole city blocks and left to ferment for years.
-This guy has been around.
-
-He listens to Steve Jackson tell his story. The dreamy
-game strategist has been dealt a bad hand. He has played
-it for all he is worth. Under his nerdish SF-fan exterior
-is a core of iron. Friends of his say Steve Jackson believes
-in the rules, believes in fair play. He will never compromise
-his principles, never give up. "Steve," Delaney says to
-Steve Jackson, "they had some balls, whoever busted you.
-You're all right!" Jackson, stunned, falls silent and
-actually blushes with pleasure.
-
-Neidorf has grown up a lot in the past year. The kid is
-a quick study, you gotta give him that. Dressed by his mom,
-the fashion manager for a national clothing chain,
-Missouri college techie-frat Craig Neidorf out-dappers
-everyone at this gig but the toniest East Coast lawyers.
-The iron jaws of prison clanged shut without him and now
-law school beckons for Neidorf. He looks like a larval Congressman.
-
-Not a "hacker," our Mr. Neidorf. He's not interested
-in computer science. Why should he be? He's not
-interested in writing C code the rest of his life,
-and besides, he's seen where the chips fall.
-To the world of computer science he and Phrack
-were just a curiosity. But to the world of law. . . .
-The kid has learned where the bodies are buried.
-He carries his notebook of press clippings wherever he goes.
-
-Phiber Optik makes fun of Neidorf for a Midwestern geek,
-for believing that "Acid Phreak" does acid and listens to acid rock.
-Hell no. Acid's never done ACID! Acid's into ACID HOUSE MUSIC.
-Jesus. The very idea of doing LSD. Our PARENTS did LSD, ya clown.
-
-Thackeray suddenly turns upon Craig Neidorf the full lighthouse
-glare of her attention and begins a determined half-hour attempt
-to WIN THE BOY OVER. The Joan of Arc of Computer Crime is
-GIVING CAREER ADVICE TO KNIGHT LIGHTNING! "Your experience
-would be very valuable--a real asset," she tells him with
-unmistakeable sixty-thousand-watt sincerity. Neidorf is fascinated.
-He listens with unfeigned attention. He's nodding and saying yes ma'am.
-Yes, Craig, you too can forget all about money and enter the glamorous
-and horribly underpaid world of PROSECUTING COMPUTER CRIME!
-You can put your former friends in prison--ooops. . . .
-
-You cannot go on dueling at modem's length indefinitely.
-You cannot beat one another senseless with rolled-up press-clippings.
-Sooner or later you have to come directly to grips.
-And yet the very act of assembling here has changed
-the entire situation drastically. John Quarterman,
-author of The Matrix, explains the Internet at his symposium.
-It is the largest news network in the world, it is growing
-by leaps and bounds, and yet you cannot measure Internet because
-you cannot stop it in place. It cannot stop, because there
-is no one anywhere in the world with the authority to stop Internet.
-It changes, yes, it grows, it embeds itself across the post-industrial,
-postmodern world and it generates community wherever it
-touches, and it is doing this all by itself.
-
-Phiber is different. A very fin de siecle kid, Phiber Optik.
-Barlow says he looks like an Edwardian dandy. He does rather.
-Shaven neck, the sides of his skull cropped hip-hop close,
-unruly tangle of black hair on top that looks pomaded,
-he stays up till four a.m. and misses all the sessions,
-then hangs out in payphone booths with his acoustic coupler
-gutsily CRACKING SYSTEMS RIGHT IN THE MIDST OF THE HEAVIEST
-LAW ENFORCEMENT DUDES IN THE U.S., or at least PRETENDING to. . . .
-Unlike "Frank Drake." Drake, who wrote Dorothy Denning out
-of nowhere, and asked for an interview for his cheapo
-cyberpunk fanzine, and then started grilling her on her ethics.
-She was squirmin', too. . . . Drake, scarecrow-tall with his
-floppy blond mohawk, rotting tennis shoes and black leather jacket
-lettered ILLUMINATI in red, gives off an unmistakeable air
-of the bohemian literatus. Drake is the kind of guy
-who reads British industrial design magazines and appreciates
-William Gibson because the quality of the prose is so tasty.
-Drake could never touch a phone or a keyboard again,
-and he'd still have the nose-ring and the blurry photocopied
-fanzines and the sampled industrial music. He's a radical punk
-with a desktop-publishing rig and an Internet address.
-Standing next to Drake, the diminutive Phiber looks like he's
-been physically coagulated out of phone-lines. Born to phreak.
-
-Dorothy Denning approaches Phiber suddenly. The two of them
-are about the same height and body-build. Denning's blue eyes
-flash behind the round window-frames of her glasses.
-"Why did you say I was `quaint?'" she asks Phiber, quaintly.
-
-It's a perfect description but Phiber is nonplussed. . .
-"Well, I uh, you know. . . ."
-
-"I also think you're quaint, Dorothy," I say, novelist to the rescue,
-the journo gift of gab. . . . She is neat and dapper and yet there's
-an arcane quality to her, something like a Pilgrim Maiden behind
-leaded glass; if she were six inches high Dorothy Denning would look
-great inside a china cabinet. . .The Cryptographeress. . .
-The Cryptographrix. . .whatever. . . . Weirdly, Peter Denning looks
-just like his wife, you could pick this gentleman out of a thousand guys
-as the soulmate of Dorothy Denning. Wearing tailored slacks,
-a spotless fuzzy varsity sweater, and a neatly knotted academician's tie. . . .
-This fineboned, exquisitely polite, utterly civilized and hyperintelligent
-couple seem to have emerged from some cleaner and finer parallel universe,
-where humanity exists to do the Brain Teasers column in Scientific American.
-Why does this Nice Lady hang out with these unsavory characters?
-
-Because the time has come for it, that's why.
-Because she's the best there is at what she does.
-
-Donn Parker is here, the Great Bald Eagle of Computer Crime. . . .
-With his bald dome, great height, and enormous Lincoln-like hands,
-the great visionary pioneer of the field plows through the lesser mortals
-like an icebreaker. . . . His eyes are fixed on the future with the
-rigidity of a bronze statue. . . . Eventually, he tells his audience,
-all business crime will be computer crime, because businesses will do
-everything through computers. "Computer crime" as a category will vanish.
-
-In the meantime, passing fads will flourish and fail and evaporate. . . .
-Parker's commanding, resonant voice is sphinxlike, everything is viewed
-from some eldritch valley of deep historical abstraction. . . .
-Yes, they've come and they've gone, these passing flaps in the world
-of digital computation. . . . The radio-frequency emanation scandal. . .
-KGB and MI5 and CIA do it every day, it's easy, but nobody else ever has. . . .
-The salami-slice fraud, mostly mythical. . . . "Crimoids," he calls them. . . .
-Computer viruses are the current crimoid champ, a lot less dangerous than
-most people let on, but the novelty is fading and there's a crimoid vacuum at
-the moment, the press is visibly hungering for something more outrageous. . . .
-The Great Man shares with us a few speculations on the coming crimoids. . . .
-Desktop Forgery! Wow. . . . Computers stolen just for the sake of the
-information within them--data-napping! Happened in Britain a while ago,
-could be the coming thing. . . . Phantom nodes in the Internet!
-
-Parker handles his overhead projector sheets with an ecclesiastical air. . . .
-He wears a grey double-breasted suit, a light blue shirt, and a
-very quiet tie of understated maroon and blue paisley. . . .
-Aphorisms emerge from him with slow, leaden emphasis. . . .
-There is no such thing as an adequately secure computer
-when one faces a sufficiently powerful adversary. . . .
-Deterrence is the most socially useful aspect of security. . . .
-People are the primary weakness in all information systems. . . .
-The entire baseline of computer security must be shifted upward. . . .
-Don't ever violate your security by publicly describing
-your security measures. . . .
-
-People in the audience are beginning to squirm, and yet
-there is something about the elemental purity of this guy's
-philosophy that compels uneasy respect. . . . Parker sounds
-like the only sane guy left in the lifeboat, sometimes.
-The guy who can prove rigorously, from deep moral principles,
-that Harvey there, the one with the broken leg and the checkered past,
-is the one who has to be, err. . .that is, Mr. Harvey is best placed
-to make the necessary sacrifice for the security and indeed
-the very survival of the rest of this lifeboat's crew. . . .
-Computer security, Parker informs us mournfully, is a
-nasty topic, and we wish we didn't have to have it. . . .
-The security expert, armed with method and logic, must think--imagine--
-everything that the adversary might do before the adversary might
-actually do it. It is as if the criminal's dark brain were an
-extensive subprogram within the shining cranium of Donn Parker.
-He is a Holmes whose Moriarty does not quite yet exist
-and so must be perfectly simulated.
-
-CFP is a stellar gathering, with the giddiness of a wedding.
-It is a happy time, a happy ending, they know their world
-is changing forever tonight, and they're proud to have been there
-to see it happen, to talk, to think, to help.
-
-And yet as night falls, a certain elegiac quality manifests itself,
-as the crowd gathers beneath the chandeliers with their wineglasses
-and dessert plates. Something is ending here, gone forever,
-and it takes a while to pinpoint it.
-
-It is the End of the Amateurs.
-
-
-
-
-
-
-
-
-
-End of the Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling
-
-*** END OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN ***
-
-***** This file should be named 101.txt or 101.zip *****
-This and all associated files of various formats will be found in:
- http://www.gutenberg.org/1/0/101/
-
-
-
-Updated editions will replace the previous one--the old editions will be
-renamed.
-
-Creating the works from public domain print editions means that no one
-owns a United States copyright in these works, so the Foundation (and
-you!) can copy and distribute it in the United States without permission
-and without paying copyright royalties. Special rules, set forth in the
-General Terms of Use part of this license, apply to copying and
-distributing Project Gutenberg-tm electronic works to protect the
-PROJECT GUTENBERG-tm concept and trademark. Project Gutenberg is a
-registered trademark, and may not be used if you charge for the eBooks,
-unless you receive specific permission. If you do not charge anything
-for copies of this eBook, complying with the rules is very easy. You may
-use this eBook for nearly any purpose such as creation of derivative
-works, reports, performances and research. They may be modified and
-printed and given away--you may do practically ANYTHING with public
-domain eBooks. Redistribution is subject to the trademark license,
-especially commercial redistribution.
-
-
-
-*** START: FULL LICENSE ***
-
-THE FULL PROJECT GUTENBERG LICENSE
-PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK
-
-To protect the Project Gutenberg-tm mission of promoting the free
-distribution of electronic works, by using or distributing this work
-(or any other work associated in any way with the phrase "Project
-Gutenberg"), you agree to comply with all the terms of the Full Project
-Gutenberg-tm License (available with this file or online at
-http://www.gutenberg.org/license).
-
-
-Section 1. General Terms of Use and Redistributing Project Gutenberg-tm
-electronic works
-
-1.A. By reading or using any part of this Project Gutenberg-tm
-electronic work, you indicate that you have read, understand, agree to
-and accept all the terms of this license and intellectual property
-(trademark/copyright) agreement. If you do not agree to abide by all
-the terms of this agreement, you must cease using and return or destroy
-all copies of Project Gutenberg-tm electronic works in your possession.
-If you paid a fee for obtaining a copy of or access to a Project
-Gutenberg-tm electronic work and you do not agree to be bound by the
-terms of this agreement, you may obtain a refund from the person or
-entity to whom you paid the fee as set forth in paragraph 1.E.8.
-
-1.B. "Project Gutenberg" is a registered trademark. It may only be
-used on or associated in any way with an electronic work by people who
-agree to be bound by the terms of this agreement. There are a few
-things that you can do with most Project Gutenberg-tm electronic works
-even without complying with the full terms of this agreement. See
-paragraph 1.C below. There are a lot of things you can do with Project
-Gutenberg-tm electronic works if you follow the terms of this agreement
-and help preserve free future access to Project Gutenberg-tm electronic
-works. See paragraph 1.E below.
-
-1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation"
-or PGLAF), owns a compilation copyright in the collection of Project
-Gutenberg-tm electronic works. Nearly all the individual works in the
-collection are in the public domain in the United States. If an
-individual work is in the public domain in the United States and you are
-located in the United States, we do not claim a right to prevent you from
-copying, distributing, performing, displaying or creating derivative
-works based on the work as long as all references to Project Gutenberg
-are removed. Of course, we hope that you will support the Project
-Gutenberg-tm mission of promoting free access to electronic works by
-freely sharing Project Gutenberg-tm works in compliance with the terms of
-this agreement for keeping the Project Gutenberg-tm name associated with
-the work. You can easily comply with the terms of this agreement by
-keeping this work in the same format with its attached full Project
-Gutenberg-tm License when you share it without charge with others.
-This particular work is one of the few copyrighted individual works
-included with the permission of the copyright holder. Information on
-the copyright owner for this particular work and the terms of use
-imposed by the copyright holder on this work are set forth at the
-beginning of this work.
-
-1.D. The copyright laws of the place where you are located also govern
-what you can do with this work. Copyright laws in most countries are in
-a constant state of change. If you are outside the United States, check
-the laws of your country in addition to the terms of this agreement
-before downloading, copying, displaying, performing, distributing or
-creating derivative works based on this work or any other Project
-Gutenberg-tm work. The Foundation makes no representations concerning
-the copyright status of any work in any country outside the United
-States.
-
-1.E. Unless you have removed all references to Project Gutenberg:
-
-1.E.1. The following sentence, with active links to, or other immediate
-access to, the full Project Gutenberg-tm License must appear prominently
-whenever any copy of a Project Gutenberg-tm work (any work on which the
-phrase "Project Gutenberg" appears, or with which the phrase "Project
-Gutenberg" is associated) is accessed, displayed, performed, viewed,
-copied or distributed:
-
-This eBook is for the use of anyone anywhere at no cost and with
-almost no restrictions whatsoever. You may copy it, give it away or
-re-use it under the terms of the Project Gutenberg License included
-with this eBook or online at www.gutenberg.org
-
-1.E.2. If an individual Project Gutenberg-tm electronic work is derived
-from the public domain (does not contain a notice indicating that it is
-posted with permission of the copyright holder), the work can be copied
-and distributed to anyone in the United States without paying any fees
-or charges. If you are redistributing or providing access to a work
-with the phrase "Project Gutenberg" associated with or appearing on the
-work, you must comply either with the requirements of paragraphs 1.E.1
-through 1.E.7 or obtain permission for the use of the work and the
-Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or
-1.E.9.
-
-1.E.3. If an individual Project Gutenberg-tm electronic work is posted
-with the permission of the copyright holder, your use and distribution
-must comply with both paragraphs 1.E.1 through 1.E.7 and any additional
-terms imposed by the copyright holder. Additional terms will be linked
-to the Project Gutenberg-tm License for all works posted with the
-permission of the copyright holder found at the beginning of this work.
-
-1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm
-License terms from this work, or any files containing a part of this
-work or any other work associated with Project Gutenberg-tm.
-
-1.E.5. Do not copy, display, perform, distribute or redistribute this
-electronic work, or any part of this electronic work, without
-prominently displaying the sentence set forth in paragraph 1.E.1 with
-active links or immediate access to the full terms of the Project
-Gutenberg-tm License.
-
-1.E.6. You may convert to and distribute this work in any binary,
-compressed, marked up, nonproprietary or proprietary form, including any
-word processing or hypertext form. However, if you provide access to or
-distribute copies of a Project Gutenberg-tm work in a format other than
-"Plain Vanilla ASCII" or other format used in the official version
-posted on the official Project Gutenberg-tm web site (www.gutenberg.org),
-you must, at no additional cost, fee or expense to the user, provide a
-copy, a means of exporting a copy, or a means of obtaining a copy upon
-request, of the work in its original "Plain Vanilla ASCII" or other
-form. Any alternate format must include the full Project Gutenberg-tm
-License as specified in paragraph 1.E.1.
-
-1.E.7. Do not charge a fee for access to, viewing, displaying,
-performing, copying or distributing any Project Gutenberg-tm works
-unless you comply with paragraph 1.E.8 or 1.E.9.
-
-1.E.8. You may charge a reasonable fee for copies of or providing
-access to or distributing Project Gutenberg-tm electronic works provided
-that
-
-- You pay a royalty fee of 20% of the gross profits you derive from
- the use of Project Gutenberg-tm works calculated using the method
- you already use to calculate your applicable taxes. The fee is
- owed to the owner of the Project Gutenberg-tm trademark, but he
- has agreed to donate royalties under this paragraph to the
- Project Gutenberg Literary Archive Foundation. Royalty payments
- must be paid within 60 days following each date on which you
- prepare (or are legally required to prepare) your periodic tax
- returns. Royalty payments should be clearly marked as such and
- sent to the Project Gutenberg Literary Archive Foundation at the
- address specified in Section 4, "Information about donations to
- the Project Gutenberg Literary Archive Foundation."
-
-- You provide a full refund of any money paid by a user who notifies
- you in writing (or by e-mail) within 30 days of receipt that s/he
- does not agree to the terms of the full Project Gutenberg-tm
- License. You must require such a user to return or
- destroy all copies of the works possessed in a physical medium
- and discontinue all use of and all access to other copies of
- Project Gutenberg-tm works.
-
-- You provide, in accordance with paragraph 1.F.3, a full refund of any
- money paid for a work or a replacement copy, if a defect in the
- electronic work is discovered and reported to you within 90 days
- of receipt of the work.
-
-- You comply with all other terms of this agreement for free
- distribution of Project Gutenberg-tm works.
-
-1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm
-electronic work or group of works on different terms than are set
-forth in this agreement, you must obtain permission in writing from
-both the Project Gutenberg Literary Archive Foundation and Michael
-Hart, the owner of the Project Gutenberg-tm trademark. Contact the
-Foundation as set forth in Section 3 below.
-
-1.F.
-
-1.F.1. Project Gutenberg volunteers and employees expend considerable
-effort to identify, do copyright research on, transcribe and proofread
-public domain works in creating the Project Gutenberg-tm
-collection. Despite these efforts, Project Gutenberg-tm electronic
-works, and the medium on which they may be stored, may contain
-"Defects," such as, but not limited to, incomplete, inaccurate or
-corrupt data, transcription errors, a copyright or other intellectual
-property infringement, a defective or damaged disk or other medium, a
-computer virus, or computer codes that damage or cannot be read by
-your equipment.
-
-1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right
-of Replacement or Refund" described in paragraph 1.F.3, the Project
-Gutenberg Literary Archive Foundation, the owner of the Project
-Gutenberg-tm trademark, and any other party distributing a Project
-Gutenberg-tm electronic work under this agreement, disclaim all
-liability to you for damages, costs and expenses, including legal
-fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT
-LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE
-PROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE
-TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE
-LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR
-INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH
-DAMAGE.
-
-1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a
-defect in this electronic work within 90 days of receiving it, you can
-receive a refund of the money (if any) you paid for it by sending a
-written explanation to the person you received the work from. If you
-received the work on a physical medium, you must return the medium with
-your written explanation. The person or entity that provided you with
-the defective work may elect to provide a replacement copy in lieu of a
-refund. If you received the work electronically, the person or entity
-providing it to you may choose to give you a second opportunity to
-receive the work electronically in lieu of a refund. If the second copy
-is also defective, you may demand a refund in writing without further
-opportunities to fix the problem.
-
-1.F.4. Except for the limited right of replacement or refund set forth
-in paragraph 1.F.3, this work is provided to you 'AS-IS,' WITH NO OTHER
-WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
-WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE.
-
-1.F.5. Some states do not allow disclaimers of certain implied
-warranties or the exclusion or limitation of certain types of damages.
-If any disclaimer or limitation set forth in this agreement violates the
-law of the state applicable to this agreement, the agreement shall be
-interpreted to make the maximum disclaimer or limitation permitted by
-the applicable state law. The invalidity or unenforceability of any
-provision of this agreement shall not void the remaining provisions.
-
-1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the
-trademark owner, any agent or employee of the Foundation, anyone
-providing copies of Project Gutenberg-tm electronic works in accordance
-with this agreement, and any volunteers associated with the production,
-promotion and distribution of Project Gutenberg-tm electronic works,
-harmless from all liability, costs and expenses, including legal fees,
-that arise directly or indirectly from any of the following which you do
-or cause to occur: (a) distribution of this or any Project Gutenberg-tm
-work, (b) alteration, modification, or additions or deletions to any
-Project Gutenberg-tm work, and (c) any Defect you cause.
-
-
-Section 2. Information about the Mission of Project Gutenberg-tm
-
-Project Gutenberg-tm is synonymous with the free distribution of
-electronic works in formats readable by the widest variety of computers
-including obsolete, old, middle-aged and new computers. It exists
-because of the efforts of hundreds of volunteers and donations from
-people in all walks of life.
-
-Volunteers and financial support to provide volunteers with the
-assistance they need are critical to reaching Project Gutenberg-tm's
-goals and ensuring that the Project Gutenberg-tm collection will
-remain freely available for generations to come. In 2001, the Project
-Gutenberg Literary Archive Foundation was created to provide a secure
-and permanent future for Project Gutenberg-tm and future generations.
-To learn more about the Project Gutenberg Literary Archive Foundation
-and how your efforts and donations can help, see Sections 3 and 4
-and the Foundation web page at http://www.pglaf.org.
-
-
-Section 3. Information about the Project Gutenberg Literary Archive
-Foundation
-
-The Project Gutenberg Literary Archive Foundation is a non profit
-501(c)(3) educational corporation organized under the laws of the
-state of Mississippi and granted tax exempt status by the Internal
-Revenue Service. The Foundation's EIN or federal tax identification
-number is 64-6221541. Its 501(c)(3) letter is posted at
-http://pglaf.org/fundraising. Contributions to the Project Gutenberg
-Literary Archive Foundation are tax deductible to the full extent
-permitted by U.S. federal laws and your state's laws.
-
-The Foundation's principal office is located at 4557 Melan Dr. S.
-Fairbanks, AK, 99712., but its volunteers and employees are scattered
-throughout numerous locations. Its business office is located at
-809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email
-business@pglaf.org. Email contact links and up to date contact
-information can be found at the Foundation's web site and official
-page at http://pglaf.org
-
-For additional contact information:
- Dr. Gregory B. Newby
- Chief Executive and Director
- gbnewby@pglaf.org
-
-Section 4. Information about Donations to the Project Gutenberg
-Literary Archive Foundation
-
-Project Gutenberg-tm depends upon and cannot survive without wide
-spread public support and donations to carry out its mission of
-increasing the number of public domain and licensed works that can be
-freely distributed in machine readable form accessible by the widest
-array of equipment including outdated equipment. Many small donations
-($1 to $5,000) are particularly important to maintaining tax exempt
-status with the IRS.
-
-The Foundation is committed to complying with the laws regulating
-charities and charitable donations in all 50 states of the United
-States. Compliance requirements are not uniform and it takes a
-considerable effort, much paperwork and many fees to meet and keep up
-with these requirements. We do not solicit donations in locations
-where we have not received written confirmation of compliance. To
-SEND DONATIONS or determine the status of compliance for any
-particular state visit http://pglaf.org
-
-While we cannot and do not solicit contributions from states where we
-have not met the solicitation requirements, we know of no prohibition
-against accepting unsolicited donations from donors in such states who
-approach us with offers to donate.
-
-International donations are gratefully accepted, but we cannot make
-any statements concerning tax treatment of donations received from
-outside the United States. U.S. laws alone swamp our small staff.
-
-Please check the Project Gutenberg Web pages for current donation
-methods and addresses. Donations are accepted in a number of other
-ways including checks, online payments and credit card donations.
-To donate, please visit: http://pglaf.org/donate
-
-
-Section 5. General Information About Project Gutenberg-tm electronic
-works.
-
-Professor Michael S. Hart is the originator of the Project Gutenberg-tm
-concept of a library of electronic works that could be freely shared
-with anyone. For thirty years, he produced and distributed Project
-Gutenberg-tm eBooks with only a loose network of volunteer support.
-
-Project Gutenberg-tm eBooks are often created from several printed
-editions, all of which are confirmed as Public Domain in the U.S.
-unless a copyright notice is included. Thus, we do not necessarily
-keep eBooks in compliance with any particular paper edition.
-
-Each eBook is in a subdirectory of the same number as the eBook's
-eBook number, often in several formats including plain vanilla ASCII,
-compressed (zipped), HTML and others.
-
-Corrected EDITIONS of our eBooks replace the old file and take over
-the old filename and etext number. The replaced older file is renamed.
-VERSIONS based on separate sources are treated as new eBooks receiving
-new filenames and etext numbers.
-
-Most people start at our Web site which has the main PG search facility:
-
-http://www.gutenberg.org
-
-This Web site includes information about Project Gutenberg-tm,
-including how to make donations to the Project Gutenberg Literary
-Archive Foundation, how to help produce our new eBooks, and how to
-subscribe to our email newsletter to hear about new eBooks.
-
-EBooks posted prior to November 2003, with eBook numbers BELOW #10000,
-are filed in directories based on their release date. If you want to
-download any of these eBooks directly, rather than using the regular
-search system you may utilize the following addresses and just
-download by the etext year.
-
-http://www.ibiblio.org/gutenberg/etext06
-
- (Or /etext 05, 04, 03, 02, 01, 00, 99,
- 98, 97, 96, 95, 94, 93, 92, 92, 91 or 90)
-
-EBooks posted since November 2003, with etext numbers OVER #10000, are
-filed in a different way. The year of a release date is no longer part
-of the directory path. The path is based on the etext number (which is
-identical to the filename). The path to the file is made up of single
-digits corresponding to all but the last digit in the filename. For
-example an eBook of filename 10234 would be found at:
-
-http://www.gutenberg.org/1/0/2/3/10234
-
-or filename 24689 would be found at:
-http://www.gutenberg.org/2/4/6/8/24689
-
-An alternative method of locating eBooks:
-http://www.gutenberg.org/GUTINDEX.ALL
-
-*** END: FULL LICENSE ***
diff --git a/common/src/leap/soledad/common/tests/server_state.py b/common/src/leap/soledad/common/tests/server_state.py deleted file mode 100644 index 2fe9472f..00000000 --- a/common/src/leap/soledad/common/tests/server_state.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -# server_state.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -State for servers to be used in tests. -""" - - -import os -import errno -import tempfile - - -from u1db.remote.server_state import ServerState -from leap.soledad.common.tests.util import ( - copy_sqlcipher_database_for_test, -) - - -class ServerStateForTests(ServerState): - - """Passed to a Request when it is instantiated. - - This is used to track server-side state, such as working-directory, open - databases, etc. - """ - - def __init__(self): - self._workingdir = tempfile.mkdtemp() - - def _relpath(self, relpath): - return os.path.join(self._workingdir, relpath) - - def open_database(self, path): - """Open a database at the given location.""" - from leap.soledad.client.sqlcipher import SQLCipherDatabase - return SQLCipherDatabase.open_database(path, '123', False) - - def create_database(self, path): - """Create a database at the given location.""" - from leap.soledad.client.sqlcipher import SQLCipherDatabase - return SQLCipherDatabase.open_database(path, '123', True) - - def check_database(self, path): - """Check if the database at the given location exists. - - Simply returns if it does or raises DatabaseDoesNotExist. - """ - db = self.open_database(path) - db.close() - - def ensure_database(self, path): - """Ensure database at the given location.""" - from leap.soledad.client.sqlcipher import SQLCipherDatabase - full_path = self._relpath(path) - db = SQLCipherDatabase.open_database(full_path, '123', False) - return db, db._replica_uid - - def delete_database(self, path): - """Delete database at the given location.""" - from leap.u1db.backends import sqlite_backend - full_path = self._relpath(path) - sqlite_backend.SQLiteDatabase.delete_database(full_path) - - def _copy_database(self, db): - return copy_sqlcipher_database_for_test(None, db) diff --git a/common/src/leap/soledad/common/tests/test_async.py b/common/src/leap/soledad/common/tests/test_async.py deleted file mode 100644 index 302ecc37..00000000 --- a/common/src/leap/soledad/common/tests/test_async.py +++ /dev/null @@ -1,144 +0,0 @@ -# -*- coding: utf-8 -*- -# test_async.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import os -import hashlib - -from twisted.internet import defer - -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.client import adbapi -from leap.soledad.client.sqlcipher import SQLCipherOptions - - -class ASyncSQLCipherRetryTestCase(BaseSoledadTest): - - """ - Test asynchronous SQLCipher operation. - """ - - NUM_DOCS = 5000 - - def _get_dbpool(self): - tmpdb = os.path.join(self.tempdir, "test.soledad") - opts = SQLCipherOptions(tmpdb, "secret", create=True) - return adbapi.getConnectionPool(opts) - - def _get_sample(self): - if not getattr(self, "_sample", None): - dirname = os.path.dirname(os.path.realpath(__file__)) - sample_file = os.path.join(dirname, "hacker_crackdown.txt") - with open(sample_file) as f: - self._sample = f.readlines() - return self._sample - - def test_concurrent_puts_fail_with_few_retries_and_small_timeout(self): - """ - Test if concurrent updates to the database with small timeout and - small number of retries fail with "database is locked" error. - - Many concurrent write attempts to the same sqlcipher database may fail - when the timeout is small and there are no retries. This test will - pass if any of the attempts to write the database fail. - - This test is much dependent on the environment and its result intends - to contrast with the test for the workaround for the "database is - locked" problem, which is addressed by the "test_concurrent_puts" test - below. - - If this test ever fails, it means that either (1) the platform where - you are running is it very powerful and you should try with an even - lower timeout value, or (2) the bug has been solved by a better - implementation of the underlying database pool, and thus this test - should be removed from the test suite. - """ - - old_timeout = adbapi.SQLCIPHER_CONNECTION_TIMEOUT - old_max_retries = adbapi.SQLCIPHER_MAX_RETRIES - - adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1 - adbapi.SQLCIPHER_MAX_RETRIES = 1 - - dbpool = self._get_dbpool() - - def _create_doc(doc): - return dbpool.runU1DBQuery("create_doc", doc) - - def _insert_docs(): - deferreds = [] - for i in range(self.NUM_DOCS): - payload = self._get_sample()[i] - chash = hashlib.sha256(payload).hexdigest() - doc = {"number": i, "payload": payload, 'chash': chash} - d = _create_doc(doc) - deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) - - def _errback(e): - if e.value[0].getErrorMessage() == "database is locked": - adbapi.SQLCIPHER_CONNECTION_TIMEOUT = old_timeout - adbapi.SQLCIPHER_MAX_RETRIES = old_max_retries - return defer.succeed("") - raise Exception - - d = _insert_docs() - d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) - d.addErrback(_errback) - return d - - def test_concurrent_puts(self): - """ - Test that many concurrent puts succeed. - - Currently, there's a known problem with the concurrent database pool - which is that many concurrent attempts to write to the database may - fail when the lock timeout is small and when there are no (or few) - retries. We currently workaround this problem by increasing the - timeout and the number of retries. - - Should this test ever fail, it probably means that the timeout and/or - number of retries should be increased for the platform you're running - the test. If the underlying database pool is ever fixed, then the test - above will fail and we should remove this comment from here. - """ - - dbpool = self._get_dbpool() - - def _create_doc(doc): - return dbpool.runU1DBQuery("create_doc", doc) - - def _insert_docs(): - deferreds = [] - for i in range(self.NUM_DOCS): - payload = self._get_sample()[i] - chash = hashlib.sha256(payload).hexdigest() - doc = {"number": i, "payload": payload, 'chash': chash} - d = _create_doc(doc) - deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) - - def _count_docs(results): - _, docs = results - if self.NUM_DOCS == len(docs): - return defer.succeed("") - raise Exception - - d = _insert_docs() - d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) - d.addCallback(_count_docs) - return d diff --git a/common/src/leap/soledad/common/tests/test_command.py b/common/src/leap/soledad/common/tests/test_command.py index c386bdd2..2136bb8f 100644 --- a/common/src/leap/soledad/common/tests/test_command.py +++ b/common/src/leap/soledad/common/tests/test_command.py @@ -21,10 +21,13 @@ from twisted.trial import unittest from leap.soledad.common.command import exec_validated_cmd +def validator(arg): + return True if arg is 'valid' else False + + class ExecuteValidatedCommandTest(unittest.TestCase): def test_argument_validation(self): - validator = lambda arg: True if arg is 'valid' else False status, out = exec_validated_cmd("command", "invalid arg", validator) self.assertEquals(status, 1) self.assertEquals(out, "invalid argument") diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py deleted file mode 100644 index 7ba50e11..00000000 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ /dev/null @@ -1,1445 +0,0 @@ -# -*- coding: utf-8 -*- -# test_couch.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Test ObjectStore and Couch backend bits. -""" - - -import json - -from urlparse import urljoin -from couchdb.client import Server -from uuid import uuid4 - -from testscenarios import TestWithScenarios -from twisted.trial import unittest -from mock import Mock - -from u1db import errors as u1db_errors -from u1db import SyncTarget -from u1db import vectorclock - -from leap.soledad.common import couch -from leap.soledad.common.document import ServerDocument -from leap.soledad.common.couch import errors - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.util import CouchDBTestCase -from leap.soledad.common.tests.util import make_local_db_and_target -from leap.soledad.common.tests.util import sync_via_synchronizer - -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import DatabaseBaseTests - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_common_backend`. -# ----------------------------------------------------------------------------- - -class TestCouchBackendImpl(CouchDBTestCase): - - def test__allocate_doc_id(self): - db = couch.CouchDatabase.open_database( - urljoin( - 'http://localhost:' + str(self.couch_port), - ('test-%s' % uuid4().hex) - ), - create=True, - ensure_ddocs=True) - doc_id1 = db._allocate_doc_id() - self.assertTrue(doc_id1.startswith('D-')) - self.assertEqual(34, len(doc_id1)) - int(doc_id1[len('D-'):], 16) - self.assertNotEqual(doc_id1, db._allocate_doc_id()) - self.delete_db(db._dbname) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -# ----------------------------------------------------------------------------- - -def make_couch_database_for_test(test, replica_uid): - port = str(test.couch_port) - dbname = ('test-%s' % uuid4().hex) - db = couch.CouchDatabase.open_database( - urljoin('http://localhost:' + port, dbname), - create=True, - replica_uid=replica_uid or 'test', - ensure_ddocs=True) - test.addCleanup(test.delete_db, dbname) - return db - - -def copy_couch_database_for_test(test, db): - port = str(test.couch_port) - couch_url = 'http://localhost:' + port - new_dbname = db._dbname + '_copy' - new_db = couch.CouchDatabase.open_database( - urljoin(couch_url, new_dbname), - create=True, - replica_uid=db._replica_uid or 'test') - # copy all docs - session = couch.Session() - old_couch_db = Server(couch_url, session=session)[db._dbname] - new_couch_db = Server(couch_url, session=session)[new_dbname] - for doc_id in old_couch_db: - doc = old_couch_db.get(doc_id) - # bypass u1db_config document - if doc_id == 'u1db_config': - pass - # copy design docs - elif doc_id.startswith('_design'): - del doc['_rev'] - new_couch_db.save(doc) - # copy u1db docs - elif 'u1db_rev' in doc: - new_doc = { - '_id': doc['_id'], - 'u1db_transactions': doc['u1db_transactions'], - 'u1db_rev': doc['u1db_rev'] - } - attachments = [] - if ('u1db_conflicts' in doc): - new_doc['u1db_conflicts'] = doc['u1db_conflicts'] - for c_rev in doc['u1db_conflicts']: - attachments.append('u1db_conflict_%s' % c_rev) - new_couch_db.save(new_doc) - # save conflict data - attachments.append('u1db_content') - for att_name in attachments: - att = old_couch_db.get_attachment(doc_id, att_name) - if (att is not None): - new_couch_db.put_attachment(new_doc, att, - filename=att_name) - # cleanup connections to prevent file descriptor leaking - return new_db - - -def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): - return ServerDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -COUCH_SCENARIOS = [ - ('couch', {'make_database_for_test': make_couch_database_for_test, - 'copy_database_for_test': copy_couch_database_for_test, - 'make_document_for_test': make_document_for_test, }), -] - - -class CouchTests( - TestWithScenarios, test_backends.AllDatabaseTests, CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class SoledadBackendTests( - TestWithScenarios, - test_backends.LocalDatabaseTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class CouchValidateGenNTransIdTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateGenNTransIdTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class CouchValidateSourceGenTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateSourceGenTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class CouchWithConflictsTests( - TestWithScenarios, - test_backends.LocalDatabaseWithConflictsTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -# Notice: the CouchDB backend does not have indexing capabilities, so we do -# not test indexing now. - -# class CouchIndexTests(test_backends.DatabaseIndexTests, CouchDBTestCase): -# -# scenarios = COUCH_SCENARIOS -# -# def tearDown(self): -# self.db.delete_database() -# test_backends.DatabaseIndexTests.tearDown(self) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -# ----------------------------------------------------------------------------- - -target_scenarios = [ - ('local', {'create_db_and_target': make_local_db_and_target}), ] - - -simple_doc = tests.simple_doc -nested_doc = tests.nested_doc - - -class SoledadBackendSyncTargetTests( - TestWithScenarios, - DatabaseBaseTests, - CouchDBTestCase): - - # TODO: implement _set_trace_hook(_shallow) in CouchSyncTarget so - # skipped tests can be succesfully executed. - - # whitebox true means self.db is the actual local db object - # against which the sync is performed - whitebox = True - - scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios)) - - def set_trace_hook(self, callback, shallow=False): - setter = (self.st._set_trace_hook if not shallow else - self.st._set_trace_hook_shallow) - try: - setter(callback) - except NotImplementedError: - self.skipTest("%s does not implement _set_trace_hook" - % (self.st.__class__.__name__,)) - - def setUp(self): - CouchDBTestCase.setUp(self) - # other stuff - self.db, self.st = self.create_db_and_target(self) - self.other_changes = [] - - def tearDown(self): - self.db.close() - CouchDBTestCase.tearDown(self) - - def receive_doc(self, doc, gen, trans_id): - self.other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - def test_sync_exchange_returns_many_new_docs(self): - # This test was replicated to allow dictionaries to be compared after - # JSON expansion (because one dictionary may have many different - # serialized representations). - doc = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - self.assertEqual(2, new_gen) - self.assertEqual( - [(doc.doc_id, doc.rev, json.loads(simple_doc), 1), - (doc2.doc_id, doc2.rev, json.loads(nested_doc), 2)], - [c[:-3] + (json.loads(c[-3]), c[-2]) for c in self.other_changes]) - if self.whitebox: - self.assertEqual( - self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': - [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - - def test_get_sync_target(self): - self.assertIsNot(None, self.st) - - def test_get_sync_info(self): - self.assertEqual( - ('test', 0, '', 0, ''), self.st.get_sync_info('other')) - - def test_create_doc_updates_sync_info(self): - self.assertEqual( - ('test', 0, '', 0, ''), self.st.get_sync_info('other')) - self.db.create_doc_from_json(simple_doc) - self.assertEqual(1, self.st.get_sync_info('other')[1]) - - def test_record_sync_info(self): - self.st.record_sync_info('replica', 10, 'T-transid') - self.assertEqual( - ('test', 0, '', 10, 'T-transid'), self.st.get_sync_info('replica')) - - def test_sync_exchange(self): - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', simple_doc), 10, - 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) - self.assertTransactionLog(['doc-id'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, last_trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_deleted(self): - doc = self.db.create_doc_from_json('{}') - edit_rev = 'replica:1|' + doc.rev - docs_by_gen = [ - (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, edit_rev, None, False) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_push_many(self): - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', simple_doc), 10, 'T-1'), - (self.make_document('doc-id2', 'replica:1', nested_doc), 11, - 'T-2')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) - self.assertGetDoc(self.db, 'doc-id2', 'replica:1', nested_doc, False) - self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(11, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_refuses_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - def test_sync_exchange_ignores_convergence(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - gen, txid = self.db._get_generation_info() - docs_by_gen = [ - (self.make_document(doc.doc_id, doc.rev, simple_doc), 10, 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=gen, - last_known_trans_id=txid, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual(([], 1), (self.other_changes, new_gen)) - - def test_sync_exchange_returns_new_docs(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - def test_sync_exchange_returns_deleted_docs(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) - self.assertEqual(2, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) - - def test_sync_exchange_getting_newer_docs(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): - expected = [] - - def before_whatschanged_cb(state): - if state != 'before whats_changed': - return - cont = '{"key": "cuncurrent"}' - conc_rev = self.db.put_doc( - self.make_document(doc.doc_id, 'test:1|z:2', cont)) - expected.append((doc.doc_id, conc_rev, cont, 3)) - - self.set_trace_hook(before_whatschanged_cb) - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertEqual(expected, [c[:-1] for c in self.other_changes]) - self.assertEqual(3, new_gen) - - def test_sync_exchange_with_concurrent_updates(self): - - def after_whatschanged_cb(state): - if state != 'after whats_changed': - return - self.db.create_doc_from_json('{"new": "doc"}') - - self.set_trace_hook(after_whatschanged_cb) - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - def test_sync_exchange_converged_handling(self): - doc = self.db.create_doc_from_json(simple_doc) - docs_by_gen = [ - (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), - (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, - 'T-bar')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - def test_sync_exchange_detect_incomplete_exchange(self): - def before_get_docs_explode(state): - if state != 'before get_docs': - return - raise u1db_errors.U1DBError("fail") - self.set_trace_hook(before_get_docs_explode) - # suppress traceback printing in the wsgiref server - # self.patch(simple_server.ServerHandler, - # 'log_exception', lambda h, exc_info: None) - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertRaises( - (u1db_errors.U1DBError, u1db_errors.BrokenSyncStream), - self.st.sync_exchange, [], 'other-replica', - last_known_generation=0, last_known_trans_id=None, - return_doc_cb=self.receive_doc) - - def test_sync_exchange_doc_ids(self): - sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) - if sync_exchange_doc_ids is None: - self.skipTest("sync_exchange_doc_ids not implemented") - db2 = self.create_database('test2') - doc = db2.create_doc_from_json(simple_doc) - new_gen, trans_id = sync_exchange_doc_ids( - db2, [(doc.doc_id, 10, 'T-sid')], 0, None, - return_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - self.assertTransactionLog([doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) - - def test__set_trace_hook(self): - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb) - self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ], - called) - - def test__set_trace_hook_shallow(self): - st_trace_shallow = self.st._set_trace_hook_shallow - target_st_trace_shallow = SyncTarget._set_trace_hook_shallow - same_meth = st_trace_shallow == self.st._set_trace_hook - same_fun = st_trace_shallow.im_func == target_st_trace_shallow.im_func - if (same_meth or same_fun): - # shallow same as full - expected = ['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ] - else: - expected = ['sync_exchange', 'record_sync_info'] - - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb, shallow=True) - self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(expected, called) - -sync_scenarios = [] -for name, scenario in COUCH_SCENARIOS: - scenario = dict(scenario) - scenario['do_sync'] = sync_via_synchronizer - sync_scenarios.append((name, scenario)) - scenario = dict(scenario) - - -class SoledadBackendSyncTests( - TestWithScenarios, - DatabaseBaseTests, - CouchDBTestCase): - - scenarios = sync_scenarios - - def create_database(self, replica_uid, sync_role=None): - if replica_uid == 'test' and sync_role is None: - # created up the chain by base class but unused - return None - db = self.create_database_for_role(replica_uid, sync_role) - if sync_role: - self._use_tracking[db] = (replica_uid, sync_role) - return db - - def create_database_for_role(self, replica_uid, sync_role): - # hook point for reuse - return DatabaseBaseTests.create_database(self, replica_uid) - - def copy_database(self, db, sync_role=None): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - db_copy = self.copy_database_for_test(self, db) - name, orig_sync_role = self._use_tracking[db] - self._use_tracking[db_copy] = ( - name + '(copy)', sync_role or orig_sync_role) - return db_copy - - def sync(self, db_from, db_to, trace_hook=None, - trace_hook_shallow=None): - from_name, from_sync_role = self._use_tracking[db_from] - to_name, to_sync_role = self._use_tracking[db_to] - if from_sync_role not in ('source', 'both'): - raise Exception("%s marked for %s use but used as source" % - (from_name, from_sync_role)) - if to_sync_role not in ('target', 'both'): - raise Exception("%s marked for %s use but used as target" % - (to_name, to_sync_role)) - return self.do_sync(self, db_from, db_to, trace_hook, - trace_hook_shallow) - - def setUp(self): - self.db = None - self.db1 = None - self.db2 = None - self.db3 = None - self.db1_copy = None - self.db2_copy = None - self._use_tracking = {} - DatabaseBaseTests.setUp(self) - - def tearDown(self): - for db in [ - self.db, self.db1, self.db2, - self.db3, self.db1_copy, self.db2_copy - ]: - if db is not None: - self.delete_db(db._dbname) - db.close() - DatabaseBaseTests.tearDown(self) - - def assertLastExchangeLog(self, db, expected): - log = getattr(db, '_last_exchange_log', None) - if log is None: - return - self.assertEqual(expected, log) - - def test_sync_tracks_db_generation_of_other(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertEqual( - (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [], 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 0}}) - - def test_sync_autoresolves(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(simple_doc, doc_id='doc') - rev1 = doc1.rev - doc2 = self.db2.create_doc_from_json(simple_doc, doc_id='doc') - rev2 = doc2.rev - self.sync(self.db1, self.db2) - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) - v = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) - - def test_sync_autoresolves_moar(self): - # here we test that when a database that has a conflicted document is - # the source of a sync, and the target database has a revision of the - # conflicted document that is newer than the source database's, and - # that target's database's document's content is the same as the - # source's document's conflict's, the source's document's conflict gets - # autoresolved, and the source's document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # `-------> - # a3b2 a1b2 (autoresolved) - # `-------> - # a3b2 a3b2 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - # because the conflict is on the source, sync it another time - self.sync(self.db1, self.db2) - # make sure db2 now has the exact same thing - self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) - - def test_sync_autoresolves_moar_backwards(self): - # here we test that when a database that has a conflicted document is - # the target of a sync, and the source database has a revision of the - # conflicted document that is newer than the target database's, and - # that source's database's document's content is the same as the - # target's document's conflict's, the target's document's conflict gets - # autoresolved, and the document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # <-------' - # a3b2 a3b2 (autoresolved and propagated) - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - self.db1.create_doc_from_json(simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - revc = self.db1.get_doc_conflicts('doc')[-1].rev - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db2, self.db1) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - vecc = vectorclock.VectorClockRev(revc) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - self.assertTrue(vec3.is_newer(vecc)) - # make sure db2 now has the exact same thing - self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) - - def test_sync_autoresolves_moar_backwards_three(self): - # same as autoresolves_moar_backwards, but with three databases (note - # all the syncs go in the same direction -- this is a more natural - # scenario): - # - # A B C - # a1 - - - # `-------> - # a1 a1 - - # `-------> - # a1 a1 a1 - # v v - # a2 a1b1 a1 - # `-------------------> - # a2 a1b1 a2 - # `-------> - # a2+a1b1 a2 - # v - # a2 a2+a1b1 a2c1 (same as a1b1) - # `-------------------> - # a2c1 a2+a1b1 a2c1 - # `-------> - # a2b2c1 a2b2c1 a2c1 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - self.db3 = self.create_database('test3', 'target') - self.db1.create_doc_from_json(simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - self.sync(self.db2, self.db3) - for db, content in [(self.db2, '{"hi": 42}'), - (self.db1, '{}'), - ]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db3) - self.sync(self.db2, self.db3) - # db2 and db3 now both have a doc of {}, but db2 has a - # conflict - doc = self.db2.get_doc('doc') - self.assertTrue(doc.has_conflicts) - revc = self.db2.get_doc_conflicts('doc')[-1].rev - self.assertEqual('{}', doc.get_json()) - self.assertEqual(self.db3.get_doc('doc').get_json(), doc.get_json()) - self.assertEqual(self.db3.get_doc('doc').rev, doc.rev) - # set db3 to have a doc of {hi:42} (same as db2 before the conflict) - doc = self.db3.get_doc('doc') - doc.set_json('{"hi": 42}') - self.db3.put_doc(doc) - rev3 = doc.rev - # sync it across to db1 - self.sync(self.db1, self.db3) - # db1 now has hi:42, with a rev that is newer than db2's doc - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertFalse(doc.has_conflicts) - self.assertEqual('{"hi": 42}', doc.get_json()) - VCR = vectorclock.VectorClockRev - self.assertTrue(VCR(rev1).is_newer(VCR(self.db2.get_doc('doc').rev))) - # so sync it to db2 - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db2.get_doc('doc') - self.assertFalse(doc.has_conflicts) - # db2's revision of the document is strictly newer than db1's before - # the sync, and db3's before that sync way back when - self.assertTrue(VCR(doc.rev).is_newer(VCR(rev1))) - self.assertTrue(VCR(doc.rev).is_newer(VCR(rev3))) - self.assertTrue(VCR(doc.rev).is_newer(VCR(revc))) - # make sure both dbs now have the exact same thing - self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) - - def test_sync_puts_changes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(simple_doc) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_pulls_changes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(simple_doc) - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertGetDoc(self.db1, doc.doc_id, doc.rev, simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [], 'last_known_gen': 0}, - 'return': {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) - - def test_sync_pulling_doesnt_update_other_if_changed(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(simple_doc) - # After the local side has sent its list of docs, before we start - # receiving the "targets" response, we update the local database with a - # new record. - # When we finish synchronizing, we can notice that something locally - # was updated, and we cannot tell c2 our new updated generation - - def before_get_docs(state): - if state != 'before get_docs': - return - self.db1.create_doc_from_json(simple_doc) - - self.assertEqual(0, self.sync(self.db1, self.db2, - trace_hook=before_get_docs)) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [], 'last_known_gen': 0}, - 'return': {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - # c2 should not have gotten a '_record_sync_info' call, because the - # local database had been updated more than just by the messages - # returned from c2. - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - - def test_sync_doesnt_update_other_if_nothing_pulled(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(simple_doc) - - def no_record_sync_info(state): - if state != 'record_sync_info': - return - self.fail('SyncTarget.record_sync_info was called') - self.assertEqual(1, self.sync(self.db1, self.db2, - trace_hook_shallow=no_record_sync_info)) - self.assertEqual( - 1, - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) - - def test_sync_ignores_convergence(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(simple_doc) - self.db3 = self.create_database('test3', 'target') - self.assertEqual(1, self.sync(self.db1, self.db3)) - self.assertEqual(0, self.sync(self.db2, self.db3)) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_ignores_superseded(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(simple_doc) - doc_rev1 = doc.rev - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.sync(self.db2, self.db3) - new_content = '{"key": "altval"}' - doc.set_json(new_content) - self.db1.put_doc(doc) - doc_rev2 = doc.rev - self.sync(self.db2, self.db1) - self.assertLastExchangeLog( - self.db1, - {'receive': {'docs': [(doc.doc_id, doc_rev1)], - 'source_uid': 'test2', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [(doc.doc_id, doc_rev2)], - 'last_gen': 2}}) - self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) - - def test_sync_sees_remote_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(simple_doc) - doc_id = doc1.doc_id - doc1_rev = doc1.rev - new_doc = '{"key": "altval"}' - doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) - doc2_rev = doc2.rev - self.assertTransactionLog([doc1.doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc_id, doc1_rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [(doc_id, doc2_rev)], - 'last_gen': 1}}) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) - self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) - - def test_sync_sees_remote_delete_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(simple_doc) - doc_id = doc1.doc_id - self.sync(self.db1, self.db2) - doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) - new_doc = '{"key": "altval"}' - doc1.set_json(new_doc) - self.db1.put_doc(doc1) - self.db2.delete_doc(doc2) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc_id, doc1.rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [(doc_id, doc2.rev)], - 'last_gen': 2}}) - self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) - self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, doc2.rev, None, False) - - def test_sync_local_race_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(simple_doc) - doc_id = doc.doc_id - doc1_rev = doc.rev - self.sync(self.db1, self.db2) - content1 = '{"key": "localval"}' - content2 = '{"key": "altval"}' - doc.set_json(content2) - self.db2.put_doc(doc) - doc2_rev2 = doc.rev - triggered = [] - - def after_whatschanged(state): - if state != 'after whats_changed': - return - triggered.append(True) - doc = self.make_document(doc_id, doc1_rev, content1) - self.db1.put_doc(doc) - - self.sync(self.db1, self.db2, trace_hook=after_whatschanged) - self.assertEqual([True], triggered) - self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) - - def test_sync_propagates_deletes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json(simple_doc) - doc_id = doc1.doc_id - self.sync(self.db1, self.db2) - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.db1.delete_doc(doc1) - deleted_rev = doc1.rev - self.sync(self.db1, self.db2) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db1, doc_id, deleted_rev, None, False) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, deleted_rev, None, False) - self.sync(self.db2, self.db3) - self.assertLastExchangeLog( - self.db3, - {'receive': {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test2', - 'source_gen': 2, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db3, doc_id, deleted_rev, None, False) - - def test_sync_propagates_deletes_2(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') - self.sync(self.db1, self.db2) - doc1_2 = self.db2.get_doc('the-doc') - self.db2.delete_doc(doc1_2) - self.sync(self.db1, self.db2) - self.assertGetDocIncludeDeleted( - self.db1, 'the-doc', doc1_2.rev, None, False) - - def test_sync_propagates_resolution(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - self.db3 = self.create_database('test3', 'both') - self.sync(self.db2, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db2._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.sync(self.db3, self.db1) - # update on 2 - doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') - self.db2.put_doc(doc2) - self.sync(self.db2, self.db3) - self.assertEqual(self.db3.get_doc('the-doc').rev, doc2.rev) - # update on 1 - doc1.set_json('{"a": 3}') - self.db1.put_doc(doc1) - # conflicts - self.sync(self.db2, self.db1) - self.sync(self.db3, self.db1) - self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) - self.assertTrue(self.db3.get_doc('the-doc').has_conflicts) - # resolve - conflicts = self.db2.get_doc_conflicts('the-doc') - doc4 = self.make_document('the-doc', None, '{"a": 4}') - revs = [doc.rev for doc in conflicts] - self.db2.resolve_doc(doc4, revs) - doc2 = self.db2.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc2.get_json()) - self.assertFalse(doc2.has_conflicts) - self.sync(self.db2, self.db3) - doc3 = self.db3.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc3.get_json()) - self.assertFalse(doc3.has_conflicts) - - def test_sync_supersedes_conflicts(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.create_database('test3', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') - self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') - self.sync(self.db3, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.sync(self.db3, self.db2) - self.assertEqual( - self.db2._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - doc1.set_json('{"a": 2}') - self.db1.put_doc(doc1) - self.sync(self.db3, self.db1) - # original doc1 should have been removed from conflicts - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - - def test_sync_stops_after_get_sync_info(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc) - self.sync(self.db1, self.db2) - - def put_hook(state): - self.fail("Tracehook triggered for %s" % (state,)) - - self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) - - def test_sync_detects_identical_replica_uid(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test1', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.assertRaises( - u1db_errors.InvalidReplicaUID, self.sync, self.db1, self.db2) - - def test_sync_detects_rollback_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) - - def test_sync_detects_rollback_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) - - def test_sync_detects_diverged_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, self.db3, self.db2) - - def test_sync_detects_diverged_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db2) - self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, self.db1, self.db3) - - def test_sync_detects_rollback_and_divergence_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, - self.db1_copy, self.db2) - - def test_sync_detects_rollback_and_divergence_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, - self.db1, self.db2_copy) - - def test_optional_sync_preserve_json(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - cont1 = '{"a": 2}' - cont2 = '{"b": 3}' - self.db1.create_doc_from_json(cont1, doc_id="1") - self.db2.create_doc_from_json(cont2, doc_id="2") - self.sync(self.db1, self.db2) - self.assertEqual(cont1, self.db2.get_doc("1").get_json()) - self.assertEqual(cont2, self.db1.get_doc("2").get_json()) - - -class SoledadBackendExceptionsTests(CouchDBTestCase): - - def setUp(self): - CouchDBTestCase.setUp(self) - - def create_db(self, ensure=True, dbname=None): - if not dbname: - dbname = ('test-%s' % uuid4().hex) - if dbname not in self.couch_server: - self.couch_server.create(dbname) - self.db = couch.CouchDatabase( - ('http://127.0.0.1:%d' % self.couch_port), - dbname, - ensure_ddocs=ensure) - - def tearDown(self): - self.db.delete_database() - self.db.close() - CouchDBTestCase.tearDown(self) - - def test_missing_design_doc_raises(self): - """ - Test that all methods that access design documents will raise if the - design docs are not present. - """ - self.create_db(ensure=False) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocError, - self.db.get_generation_info) - # get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocError, - self.db.get_trans_id_for_gen, 1) - # get_transaction_log() - self.assertRaises( - errors.MissingDesignDocError, - self.db.get_transaction_log) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocError, - self.db.whats_changed) - - def test_missing_design_doc_functions_raises(self): - """ - Test that all methods that access design documents list functions - will raise if the functions are not present. - """ - self.create_db(ensure=True) - # erase views from _design/transactions - transactions = self.db._database['_design/transactions'] - transactions['lists'] = {} - self.db._database.save(transactions) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_generation_info) - # get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_trans_id_for_gen, 1) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.whats_changed) - - def test_absent_design_doc_functions_raises(self): - """ - Test that all methods that access design documents list functions - will raise if the functions are not present. - """ - self.create_db(ensure=True) - # erase views from _design/transactions - transactions = self.db._database['_design/transactions'] - del transactions['lists'] - self.db._database.save(transactions) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_generation_info) - # _get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_trans_id_for_gen, 1) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.whats_changed) - - def test_missing_design_doc_named_views_raises(self): - """ - Test that all methods that access design documents' named views will - raise if the views are not present. - """ - self.create_db(ensure=True) - # erase views from _design/docs - docs = self.db._database['_design/docs'] - del docs['views'] - self.db._database.save(docs) - # erase views from _design/syncs - syncs = self.db._database['_design/syncs'] - del syncs['views'] - self.db._database.save(syncs) - # erase views from _design/transactions - transactions = self.db._database['_design/transactions'] - del transactions['views'] - self.db._database.save(transactions) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.get_generation_info) - # _get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.get_trans_id_for_gen, 1) - # _get_transaction_log() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.get_transaction_log) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.whats_changed) - - def test_deleted_design_doc_raises(self): - """ - Test that all methods that access design documents will raise if the - design docs are not present. - """ - self.create_db(ensure=True) - # delete _design/docs - del self.db._database['_design/docs'] - # delete _design/syncs - del self.db._database['_design/syncs'] - # delete _design/transactions - del self.db._database['_design/transactions'] - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_generation_info) - # get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_trans_id_for_gen, 1) - # get_transaction_log() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_transaction_log) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.whats_changed) - - def test_ensure_ddoc_independently(self): - """ - Test that a missing ddocs other than _design/docs will be ensured - even if _design/docs is there. - """ - self.create_db(ensure=True) - del self.db._database['_design/transactions'] - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_transaction_log) - self.create_db(ensure=True, dbname=self.db._dbname) - self.db.get_transaction_log() - - def test_ensure_security_doc(self): - """ - Ensure_security creates a _security ddoc to ensure that only soledad - will have the lowest privileged access to an user db. - """ - self.create_db(ensure=False) - self.assertFalse(self.db._database.resource.get_json('_security')[2]) - self.db.ensure_security_ddoc() - security_ddoc = self.db._database.resource.get_json('_security')[2] - self.assertIn('admins', security_ddoc) - self.assertFalse(security_ddoc['admins']['names']) - self.assertIn('members', security_ddoc) - self.assertIn('soledad', security_ddoc['members']['names']) - - def test_ensure_security_from_configuration(self): - """ - Given a configuration, follow it to create the security document - """ - self.create_db(ensure=False) - configuration = {'members': ['user1', 'user2'], - 'members_roles': ['role1', 'role2'], - 'admins': ['admin'], - 'admins_roles': ['administrators'] - } - self.db.ensure_security_ddoc(configuration) - - security_ddoc = self.db._database.resource.get_json('_security')[2] - self.assertEquals(configuration['admins'], - security_ddoc['admins']['names']) - self.assertEquals(configuration['admins_roles'], - security_ddoc['admins']['roles']) - self.assertEquals(configuration['members'], - security_ddoc['members']['names']) - self.assertEquals(configuration['members_roles'], - security_ddoc['members']['roles']) - - -class DatabaseNameValidationTest(unittest.TestCase): - - def test_database_name_validation(self): - inject = couch.state.is_db_name_valid("user-deadbeef | cat /secret") - self.assertFalse(inject) - self.assertTrue(couch.state.is_db_name_valid("user-cafe1337")) - - -class CommandBasedDBCreationTest(unittest.TestCase): - - def test_ensure_db_using_custom_command(self): - state = couch.state.CouchServerState("url", create_cmd="echo") - mock_db = Mock() - mock_db.replica_uid = 'replica_uid' - state.open_database = Mock(return_value=mock_db) - db, replica_uid = state.ensure_database("user-1337") # works - self.assertEquals(mock_db, db) - self.assertEquals(mock_db.replica_uid, replica_uid) - - def test_raises_unauthorized_on_failure(self): - state = couch.state.CouchServerState("url", create_cmd="inexistent") - self.assertRaises(u1db_errors.Unauthorized, - state.ensure_database, "user-1337") - - def test_raises_unauthorized_by_default(self): - state = couch.state.CouchServerState("url") - self.assertRaises(u1db_errors.Unauthorized, - state.ensure_database, "user-1337") diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py deleted file mode 100644 index 8cd3ae08..00000000 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ /dev/null @@ -1,371 +0,0 @@ -# -*- coding: utf-8 -*- -# test_couch_operations_atomicity.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test atomicity of couch operations. -""" -import os -import tempfile -import threading - -from urlparse import urljoin -from twisted.internet import defer -from uuid import uuid4 - -from leap.soledad.client import Soledad -from leap.soledad.common.couch.state import CouchServerState -from leap.soledad.common.couch import CouchDatabase - -from leap.soledad.common.tests.util import ( - make_token_soledad_app, - make_soledad_document_for_test, - soledad_sync_target, -) -from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer - - -REPEAT_TIMES = 20 - - -class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): - - @staticmethod - def make_app_after_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def _soledad_instance(self, user=None, passphrase=u'123', - prefix='', - secrets_path='secrets.json', - local_db_path='soledad.u1db', server_url='', - cert_file=None, auth_token=None): - """ - Instantiate Soledad. - """ - user = user or self.user - - # this callback ensures we save a document which is sent to the shared - # db. - def _put_doc_side_effect(doc): - self._doc_put = doc - - soledad = Soledad( - user, - passphrase, - secrets_path=os.path.join(self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, - cert_file=cert_file, - auth_token=auth_token, - shared_db=self.get_default_shared_mock(_put_doc_side_effect)) - self.addCleanup(soledad.close) - return soledad - - def make_app(self): - self.request_state = CouchServerState(self.couch_url) - return self.make_app_after_state(self.request_state) - - def setUp(self): - TestCaseWithServer.setUp(self) - CouchDBTestCase.setUp(self) - self.user = ('user-%s' % uuid4().hex) - self.db = CouchDatabase.open_database( - urljoin(self.couch_url, 'user-' + self.user), - create=True, - replica_uid='replica', - ensure_ddocs=True) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.startTwistedServer() - - def tearDown(self): - self.db.delete_database() - self.db.close() - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - - # - # Sequential tests - # - - def test_correct_transaction_log_after_sequential_puts(self): - """ - Assert that the transaction_log increases accordingly with sequential - puts. - """ - doc = self.db.create_doc({'ops': 0}) - docs = [doc.doc_id] - for i in range(0, REPEAT_TIMES): - self.assertEqual( - i + 1, len(self.db._get_transaction_log())) - doc.content['ops'] += 1 - self.db.put_doc(doc) - docs.append(doc.doc_id) - - # assert length of transaction_log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - REPEAT_TIMES + 1, len(transaction_log)) - - # assert that all entries in the log belong to the same doc - self.assertEqual(REPEAT_TIMES + 1, len(docs)) - for doc_id in docs: - self.assertEqual( - REPEAT_TIMES + 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - def test_correct_transaction_log_after_sequential_deletes(self): - """ - Assert that the transaction_log increases accordingly with sequential - puts and deletes. - """ - docs = [] - for i in range(0, REPEAT_TIMES): - doc = self.db.create_doc({'ops': 0}) - self.assertEqual( - 2 * i + 1, len(self.db._get_transaction_log())) - docs.append(doc.doc_id) - self.db.delete_doc(doc) - self.assertEqual( - 2 * i + 2, len(self.db._get_transaction_log())) - - # assert length of transaction_log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - 2 * REPEAT_TIMES, len(transaction_log)) - - # assert that each doc appears twice in the transaction_log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 2, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - @defer.inlineCallbacks - def test_correct_sync_log_after_sequential_syncs(self): - """ - Assert that the sync_log increases accordingly with sequential syncs. - """ - sol = self._soledad_instance( - auth_token='auth-token', - server_url=self.getURL()) - source_replica_uid = sol._dbpool.replica_uid - - def _create_docs(): - deferreds = [] - for i in xrange(0, REPEAT_TIMES): - deferreds.append(sol.create_doc({})) - return defer.gatherResults(deferreds) - - def _assert_transaction_and_sync_logs(results, sync_idx): - # assert sizes of transaction and sync logs - self.assertEqual( - sync_idx * REPEAT_TIMES, - len(self.db._get_transaction_log())) - gen, _ = self.db._get_replica_gen_and_trans_id(source_replica_uid) - self.assertEqual(sync_idx * REPEAT_TIMES, gen) - - def _assert_sync(results, sync_idx): - gen, docs = results - self.assertEqual((sync_idx + 1) * REPEAT_TIMES, gen) - self.assertEqual((sync_idx + 1) * REPEAT_TIMES, len(docs)) - # assert sizes of transaction and sync logs - self.assertEqual((sync_idx + 1) * REPEAT_TIMES, - len(self.db._get_transaction_log())) - target_known_gen, target_known_trans_id = \ - self.db._get_replica_gen_and_trans_id(source_replica_uid) - # assert it has the correct gen and trans_id - conn_key = sol._dbpool._u1dbconnections.keys().pop() - conn = sol._dbpool._u1dbconnections[conn_key] - sol_gen, sol_trans_id = conn._get_generation_info() - self.assertEqual(sol_gen, target_known_gen) - self.assertEqual(sol_trans_id, target_known_trans_id) - - # sync first time and assert success - results = yield _create_docs() - _assert_transaction_and_sync_logs(results, 0) - yield sol.sync() - results = yield sol.get_all_docs() - _assert_sync(results, 0) - - # create more docs, sync second time and assert success - results = yield _create_docs() - _assert_transaction_and_sync_logs(results, 1) - yield sol.sync() - results = yield sol.get_all_docs() - _assert_sync(results, 1) - - # - # Concurrency tests - # - - class _WorkerThread(threading.Thread): - - def __init__(self, params, run_method): - threading.Thread.__init__(self) - self._params = params - self._run_method = run_method - - def run(self): - self._run_method(self) - - def test_correct_transaction_log_after_concurrent_puts(self): - """ - Assert that the transaction_log increases accordingly with concurrent - puts. - """ - pool = threading.BoundedSemaphore(value=1) - threads = [] - docs = [] - - def _run_method(self): - doc = self._params['db'].create_doc({}) - pool.acquire() - self._params['docs'].append(doc.doc_id) - pool.release() - - for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread( - {'docs': docs, 'db': self.db}, - _run_method) - thread.start() - threads.append(thread) - - for thread in threads: - thread.join() - - # assert length of transaction_log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - REPEAT_TIMES, len(transaction_log)) - - # assert all documents are in the log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - def test_correct_transaction_log_after_concurrent_deletes(self): - """ - Assert that the transaction_log increases accordingly with concurrent - puts and deletes. - """ - threads = [] - docs = [] - pool = threading.BoundedSemaphore(value=1) - - # create/delete method that will be run concurrently - def _run_method(self): - doc = self._params['db'].create_doc({}) - pool.acquire() - docs.append(doc.doc_id) - pool.release() - self._params['db'].delete_doc(doc) - - # launch concurrent threads - for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread({'db': self.db}, _run_method) - thread.start() - threads.append(thread) - - # wait for threads to finish - for thread in threads: - thread.join() - - # assert transaction log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - 2 * REPEAT_TIMES, len(transaction_log)) - # assert that each doc appears twice in the transaction_log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 2, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - def test_correct_sync_log_after_concurrent_puts_and_sync(self): - """ - Assert that the sync_log is correct after concurrent syncs. - """ - docs = [] - - sol = self._soledad_instance( - auth_token='auth-token', - server_url=self.getURL()) - - def _save_doc_ids(results): - for doc in results: - docs.append(doc.doc_id) - - # create documents in parallel - deferreds = [] - for i in range(0, REPEAT_TIMES): - d = sol.create_doc({}) - deferreds.append(d) - - # wait for documents creation and sync - d = defer.gatherResults(deferreds) - d.addCallback(_save_doc_ids) - d.addCallback(lambda _: sol.sync()) - - def _assert_logs(results): - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - d.addCallback(_assert_logs) - d.addCallback(lambda _: sol.close()) - - return d - - @defer.inlineCallbacks - def test_concurrent_syncs_do_not_fail(self): - """ - Assert that concurrent attempts to sync end up being executed - sequentially and do not fail. - """ - docs = [] - - sol = self._soledad_instance( - auth_token='auth-token', - server_url=self.getURL()) - - deferreds = [] - for i in xrange(0, REPEAT_TIMES): - d = sol.create_doc({}) - d.addCallback(lambda doc: docs.append(doc.doc_id)) - d.addCallback(lambda _: sol.sync()) - deferreds.append(d) - yield defer.gatherResults(deferreds, consumeErrors=True) - - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py deleted file mode 100644 index ca10a1e1..00000000 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ /dev/null @@ -1,225 +0,0 @@ -# -*- coding: utf-8 -*- -# test_crypto.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Tests for cryptographic related stuff. -""" -import os -import hashlib -import binascii - -from leap.soledad.client import crypto -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.crypto import WrongMacError -from leap.soledad.common.crypto import UnknownMacMethodError -from leap.soledad.common.crypto import EncryptionMethods -from leap.soledad.common.crypto import ENC_JSON_KEY -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.common.crypto import MAC_KEY -from leap.soledad.common.crypto import MAC_METHOD_KEY - - -class EncryptedSyncTestCase(BaseSoledadTest): - - """ - Tests that guarantee that data will always be encrypted when syncing. - """ - - def test_encrypt_decrypt_json(self): - """ - Test encrypting and decrypting documents. - """ - simpledoc = {'key': 'val'} - doc1 = SoledadDocument(doc_id='id') - doc1.content = simpledoc - - # encrypt doc - doc1.set_json(self._soledad._crypto.encrypt_doc(doc1)) - # assert content is different and includes keys - self.assertNotEqual( - simpledoc, doc1.content, - 'incorrect document encryption') - self.assertTrue(ENC_JSON_KEY in doc1.content) - self.assertTrue(ENC_SCHEME_KEY in doc1.content) - # decrypt doc - doc1.set_json(self._soledad._crypto.decrypt_doc(doc1)) - self.assertEqual( - simpledoc, doc1.content, 'incorrect document encryption') - - -class RecoveryDocumentTestCase(BaseSoledadTest): - - def test_export_recovery_document_raw(self): - rd = self._soledad.secrets._export_recovery_document() - secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0] - # assert exported secret is the same - secret = self._soledad.secrets._decrypt_storage_secret( - rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]) - self.assertEqual(secret_id, self._soledad.secrets._secret_id) - self.assertEqual(secret, self._soledad.secrets._secrets[secret_id]) - # assert recovery document structure - encrypted_secret = rd[ - self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] - self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret) - self.assertTrue( - encrypted_secret[self._soledad.secrets.CIPHER_KEY] == 'aes256') - self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret) - self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) - - def test_import_recovery_document(self): - rd = self._soledad.secrets._export_recovery_document() - s = self._soledad_instance() - s.secrets._import_recovery_document(rd) - s.secrets.set_secret_id(self._soledad.secrets._secret_id) - self.assertEqual(self._soledad.storage_secret, - s.storage_secret, - 'Failed settinng secret for symmetric encryption.') - s.close() - - -class SoledadSecretsTestCase(BaseSoledadTest): - - def test__gen_secret(self): - # instantiate and save secret_id - sol = self._soledad_instance(user='user@leap.se') - self.assertTrue(len(sol.secrets._secrets) == 1) - secret_id_1 = sol.secrets.secret_id - # assert id is hash of secret - self.assertTrue( - secret_id_1 == hashlib.sha256(sol.storage_secret).hexdigest()) - # generate new secret - secret_id_2 = sol.secrets._gen_secret() - self.assertTrue(secret_id_1 != secret_id_2) - sol.close() - # re-instantiate - sol = self._soledad_instance(user='user@leap.se') - sol.secrets.set_secret_id(secret_id_1) - # assert ids are valid - self.assertTrue(len(sol.secrets._secrets) == 2) - self.assertTrue(secret_id_1 in sol.secrets._secrets) - self.assertTrue(secret_id_2 in sol.secrets._secrets) - # assert format of secret 1 - self.assertTrue(sol.storage_secret is not None) - self.assertIsInstance(sol.storage_secret, str) - secret_length = sol.secrets.GEN_SECRET_LENGTH - self.assertTrue(len(sol.storage_secret) == secret_length) - # assert format of secret 2 - sol.secrets.set_secret_id(secret_id_2) - self.assertTrue(sol.storage_secret is not None) - self.assertIsInstance(sol.storage_secret, str) - self.assertTrue(len(sol.storage_secret) == secret_length) - # assert id is hash of new secret - self.assertTrue( - secret_id_2 == hashlib.sha256(sol.storage_secret).hexdigest()) - sol.close() - - def test__has_secret(self): - sol = self._soledad_instance( - user='user@leap.se', prefix=self.rand_prefix) - self.assertTrue( - sol.secrets._has_secret(), - "Should have a secret at this point") - # setting secret id to None should not interfere in the fact we have a - # secret. - sol.secrets.set_secret_id(None) - self.assertTrue( - sol.secrets._has_secret(), - "Should have a secret at this point") - # but not being able to decrypt correctly should - sol.secrets._secrets[sol.secrets.secret_id] = None - self.assertFalse(sol.secrets._has_secret()) - sol.close() - - -class MacAuthTestCase(BaseSoledadTest): - - def test_decrypt_with_wrong_mac_raises(self): - """ - Trying to decrypt a document with wrong MAC should raise. - """ - simpledoc = {'key': 'val'} - doc = SoledadDocument(doc_id='id') - doc.content = simpledoc - # encrypt doc - doc.set_json(self._soledad._crypto.encrypt_doc(doc)) - self.assertTrue(MAC_KEY in doc.content) - self.assertTrue(MAC_METHOD_KEY in doc.content) - # mess with MAC - doc.content[MAC_KEY] = '1234567890ABCDEF' - # try to decrypt doc - self.assertRaises( - WrongMacError, - self._soledad._crypto.decrypt_doc, doc) - - def test_decrypt_with_unknown_mac_method_raises(self): - """ - Trying to decrypt a document with unknown MAC method should raise. - """ - simpledoc = {'key': 'val'} - doc = SoledadDocument(doc_id='id') - doc.content = simpledoc - # encrypt doc - doc.set_json(self._soledad._crypto.encrypt_doc(doc)) - self.assertTrue(MAC_KEY in doc.content) - self.assertTrue(MAC_METHOD_KEY in doc.content) - # mess with MAC method - doc.content[MAC_METHOD_KEY] = 'mymac' - # try to decrypt doc - self.assertRaises( - UnknownMacMethodError, - self._soledad._crypto.decrypt_doc, doc) - - -class SoledadCryptoAESTestCase(BaseSoledadTest): - - def test_encrypt_decrypt_sym(self): - # generate 256-bit key - key = os.urandom(32) - iv, cyphertext = crypto.encrypt_sym('data', key) - self.assertTrue(cyphertext is not None) - self.assertTrue(cyphertext != '') - self.assertTrue(cyphertext != 'data') - plaintext = crypto.decrypt_sym(cyphertext, key, iv) - self.assertEqual('data', plaintext) - - def test_decrypt_with_wrong_iv_fails(self): - key = os.urandom(32) - iv, cyphertext = crypto.encrypt_sym('data', key) - self.assertTrue(cyphertext is not None) - self.assertTrue(cyphertext != '') - self.assertTrue(cyphertext != 'data') - # get a different iv by changing the first byte - rawiv = binascii.a2b_base64(iv) - wrongiv = rawiv - while wrongiv == rawiv: - wrongiv = os.urandom(1) + rawiv[1:] - plaintext = crypto.decrypt_sym( - cyphertext, key, iv=binascii.b2a_base64(wrongiv)) - self.assertNotEqual('data', plaintext) - - def test_decrypt_with_wrong_key_fails(self): - key = os.urandom(32) - iv, cyphertext = crypto.encrypt_sym('data', key) - self.assertTrue(cyphertext is not None) - self.assertTrue(cyphertext != '') - self.assertTrue(cyphertext != 'data') - wrongkey = os.urandom(32) # 256-bits key - # ensure keys are different in case we are extremely lucky - while wrongkey == key: - wrongkey = os.urandom(32) - plaintext = crypto.decrypt_sym(cyphertext, wrongkey, iv) - self.assertNotEqual('data', plaintext) diff --git a/common/src/leap/soledad/common/tests/test_encdecpool.py b/common/src/leap/soledad/common/tests/test_encdecpool.py deleted file mode 100644 index 694eb7ad..00000000 --- a/common/src/leap/soledad/common/tests/test_encdecpool.py +++ /dev/null @@ -1,243 +0,0 @@ -# -*- coding: utf-8 -*- -# test_encdecpool.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Tests for encryption and decryption pool. -""" -import json -from random import shuffle - -from twisted.internet.defer import inlineCallbacks - -from leap.soledad.client.encdecpool import SyncEncrypterPool -from leap.soledad.client.encdecpool import SyncDecrypterPool - -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.tests.util import BaseSoledadTest - - -DOC_ID = "mydoc" -DOC_REV = "rev" -DOC_CONTENT = {'simple': 'document'} - - -class TestSyncEncrypterPool(BaseSoledadTest): - - def setUp(self): - BaseSoledadTest.setUp(self) - crypto = self._soledad._crypto - sync_db = self._soledad._sync_db - self._pool = SyncEncrypterPool(crypto, sync_db) - self._pool.start() - - def tearDown(self): - self._pool.stop() - BaseSoledadTest.tearDown(self) - - @inlineCallbacks - def test_get_encrypted_doc_returns_none(self): - """ - Test that trying to get an encrypted doc from the pool returns None if - the document was never added for encryption. - """ - doc = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) - self.assertIsNone(doc) - - @inlineCallbacks - def test_enqueue_doc_for_encryption_and_get_encrypted_doc(self): - """ - Test that the pool actually encrypts a document added to the queue. - """ - doc = SoledadDocument( - doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - self._pool.enqueue_doc_for_encryption(doc) - - # exhaustivelly attempt to get the encrypted document - encrypted = None - attempts = 0 - while encrypted is None and attempts < 10: - encrypted = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) - attempts += 1 - - self.assertIsNotNone(encrypted) - self.assertTrue(attempts < 10) - - -class TestSyncDecrypterPool(BaseSoledadTest): - - def _insert_doc_cb(self, doc, gen, trans_id): - """ - Method used to mock the sync's return_doc_cb callback. - """ - self._inserted_docs.append((doc, gen, trans_id)) - - def setUp(self): - BaseSoledadTest.setUp(self) - # setup the pool - self._pool = SyncDecrypterPool( - self._soledad._crypto, - self._soledad._sync_db, - source_replica_uid=self._soledad._dbpool.replica_uid, - insert_doc_cb=self._insert_doc_cb) - # reset the inserted docs mock - self._inserted_docs = [] - - def tearDown(self): - if self._pool.running: - self._pool.stop() - BaseSoledadTest.tearDown(self) - - def test_insert_received_doc(self): - """ - Test that one document added to the pool is inserted using the - callback. - """ - self._pool.start(1) - self._pool.insert_received_doc( - DOC_ID, DOC_REV, "{}", 1, "trans_id", 1) - - def _assert_doc_was_inserted(_): - self.assertEqual( - self._inserted_docs, - [(SoledadDocument(DOC_ID, DOC_REV, "{}"), 1, u"trans_id")]) - - self._pool.deferred.addCallback(_assert_doc_was_inserted) - return self._pool.deferred - - def test_insert_received_doc_many(self): - """ - Test that many documents added to the pool are inserted using the - callback. - """ - many = 100 - self._pool.start(many) - - # insert many docs in the pool - for i in xrange(many): - gen = idx = i + 1 - doc_id = "doc_id: %d" % idx - rev = "rev: %d" % idx - content = {'idx': idx} - trans_id = "trans_id: %d" % idx - self._pool.insert_received_doc( - doc_id, rev, content, gen, trans_id, idx) - - def _assert_doc_was_inserted(_): - self.assertEqual(many, len(self._inserted_docs)) - idx = 1 - for doc, gen, trans_id in self._inserted_docs: - expected_gen = idx - expected_doc_id = "doc_id: %d" % idx - expected_rev = "rev: %d" % idx - expected_content = json.dumps({'idx': idx}) - expected_trans_id = "trans_id: %d" % idx - - self.assertEqual(expected_doc_id, doc.doc_id) - self.assertEqual(expected_rev, doc.rev) - self.assertEqual(expected_content, json.dumps(doc.content)) - self.assertEqual(expected_gen, gen) - self.assertEqual(expected_trans_id, trans_id) - - idx += 1 - - self._pool.deferred.addCallback(_assert_doc_was_inserted) - return self._pool.deferred - - def test_insert_encrypted_received_doc(self): - """ - Test that one encrypted document added to the pool is decrypted and - inserted using the callback. - """ - crypto = self._soledad._crypto - doc = SoledadDocument( - doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - - # insert the encrypted document in the pool - self._pool.start(1) - self._pool.insert_encrypted_received_doc( - DOC_ID, DOC_REV, encrypted_content, 1, "trans_id", 1) - - def _assert_doc_was_decrypted_and_inserted(_): - self.assertEqual(1, len(self._inserted_docs)) - self.assertEqual(self._inserted_docs, [(doc, 1, u"trans_id")]) - - self._pool.deferred.addCallback( - _assert_doc_was_decrypted_and_inserted) - return self._pool.deferred - - def test_insert_encrypted_received_doc_many(self, many=100): - """ - Test that many encrypted documents added to the pool are decrypted and - inserted using the callback. - """ - crypto = self._soledad._crypto - self._pool.start(many) - docs = [] - - # insert many encrypted docs in the pool - for i in xrange(many): - gen = idx = i + 1 - doc_id = "doc_id: %d" % idx - rev = "rev: %d" % idx - content = {'idx': idx} - trans_id = "trans_id: %d" % idx - - doc = SoledadDocument( - doc_id=doc_id, rev=rev, json=json.dumps(content)) - - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - docs.append((doc_id, rev, encrypted_content, gen, - trans_id, idx)) - shuffle(docs) - - for doc in docs: - self._pool.insert_encrypted_received_doc(*doc) - - def _assert_docs_were_decrypted_and_inserted(_): - self.assertEqual(many, len(self._inserted_docs)) - idx = 1 - for doc, gen, trans_id in self._inserted_docs: - expected_gen = idx - expected_doc_id = "doc_id: %d" % idx - expected_rev = "rev: %d" % idx - expected_content = json.dumps({'idx': idx}) - expected_trans_id = "trans_id: %d" % idx - - self.assertEqual(expected_doc_id, doc.doc_id) - self.assertEqual(expected_rev, doc.rev) - self.assertEqual(expected_content, json.dumps(doc.content)) - self.assertEqual(expected_gen, gen) - self.assertEqual(expected_trans_id, trans_id) - - idx += 1 - - self._pool.deferred.addCallback( - _assert_docs_were_decrypted_and_inserted) - return self._pool.deferred - - @inlineCallbacks - def test_pool_reuse(self): - """ - The pool is reused between syncs, this test verifies that - reusing is fine. - """ - for i in xrange(3): - yield self.test_insert_encrypted_received_doc_many(5) - self._inserted_docs = [] - decrypted_docs = yield self._pool._get_docs(encrypted=False) - # check that decrypted docs staging is clean - self.assertEquals([], decrypted_docs) diff --git a/common/src/leap/soledad/common/tests/test_http.py b/common/src/leap/soledad/common/tests/test_http.py deleted file mode 100644 index bc486fe3..00000000 --- a/common/src/leap/soledad/common/tests/test_http.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# test_http.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test Leap backend bits: test http database -""" -from u1db.remote import http_database - -from leap.soledad.client import auth -from leap.soledad.common.tests.u1db_tests import test_http_database - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_database`. -# ----------------------------------------------------------------------------- - -class _HTTPDatabase(http_database.HTTPDatabase, auth.TokenBasedAuth): - - """ - Wraps our token auth implementation. - """ - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - -class TestHTTPDatabaseWithCreds( - test_http_database.TestHTTPDatabaseCtrWithCreds): - - def test_get_sync_target_inherits_token_credentials(self): - # this test was from TestDatabaseSimpleOperations but we put it here - # for convenience. - self.db = _HTTPDatabase('dbase') - self.db.set_token_credentials('user-uuid', 'auth-token') - st = self.db.get_sync_target() - self.assertEqual(self.db._creds, st._creds) - - def test_ctr_with_creds(self): - db1 = _HTTPDatabase('http://dbs/db', creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - self.assertIn('token', db1._creds) diff --git a/common/src/leap/soledad/common/tests/test_http_client.py b/common/src/leap/soledad/common/tests/test_http_client.py deleted file mode 100644 index 700ae3b6..00000000 --- a/common/src/leap/soledad/common/tests/test_http_client.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- -# test_http_client.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test Leap backend bits: sync target -""" -import json - -from u1db.remote import http_client - -from testscenarios import TestWithScenarios - -from leap.soledad.client import auth -from leap.soledad.common.tests.u1db_tests import test_http_client -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_client`. -# ----------------------------------------------------------------------------- - -class TestSoledadClientBase( - TestWithScenarios, - test_http_client.TestHTTPClientBase): - - """ - This class should be used to test Token auth. - """ - - def getClientWithToken(self, **kwds): - self.startServer() - - class _HTTPClientWithToken( - http_client.HTTPClientBase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - return _HTTPClientWithToken(self.getURL('dbase'), **kwds) - - def test_oauth(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def test_oauth_ctr_creds(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def test_oauth_Unauthorized(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def app(self, environ, start_response): - res = test_http_client.TestHTTPClientBase.app( - self, environ, start_response) - if res is not None: - return res - # mime solead application here. - if '/token' in environ['PATH_INFO']: - auth = environ.get(SoledadTokenAuthMiddleware.HTTP_AUTH_KEY) - if not auth: - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - scheme, encoded = auth.split(None, 1) - if scheme.lower() != 'token': - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - uuid, token = encoded.decode('base64').split(':', 1) - if uuid != 'user-uuid' and token != 'auth-token': - return Exception("Incorrect address or token.") - start_response("200 OK", [('Content-Type', 'application/json')]) - return [json.dumps([environ['PATH_INFO'], uuid, token])] - - def test_token(self): - """ - Test if token is sent correctly. - """ - cli = self.getClientWithToken() - cli.set_token_credentials('user-uuid', 'auth-token') - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - - def test_token_ctr_creds(self): - cli = self.getClientWithToken(creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) diff --git a/common/src/leap/soledad/common/tests/test_https.py b/common/src/leap/soledad/common/tests/test_https.py deleted file mode 100644 index eeeb4982..00000000 --- a/common/src/leap/soledad/common/tests/test_https.py +++ /dev/null @@ -1,138 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sync_target.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Test Leap backend bits: https -""" - - -from u1db.remote import http_client - -from leap.soledad import client - -from testscenarios import TestWithScenarios - -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import test_https -from leap.soledad.common.tests.util import ( - BaseSoledadTest, - make_soledad_document_for_test, - make_soledad_app, - make_token_soledad_app, -) - - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_soledad_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_https`. -# ----------------------------------------------------------------------------- - -def token_leap_https_sync_target(test, host, path, cert_file=None): - _, port = test.server.server_address - # source_replica_uid = test._soledad._dbpool.replica_uid - creds = {'token': {'uuid': 'user-uuid', 'token': 'auth-token'}} - if not cert_file: - cert_file = test.cacert_pem - st = client.http_target.SoledadHTTPSyncTarget( - 'https://%s:%d/%s' % (host, port, path), - source_replica_uid='other-id', - creds=creds, - crypto=test._soledad._crypto, - cert_file=cert_file) - return st - - -class TestSoledadHTTPSyncTargetHttpsSupport( - TestWithScenarios, - test_https.TestHttpSyncTargetHttpsSupport, - BaseSoledadTest): - - scenarios = [ - ('token_soledad_https', - {'server_def': test_https.https_server_def, - 'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_soledad_document_for_test, - 'sync_target': token_leap_https_sync_target}), - ] - - def setUp(self): - # the parent constructor undoes our SSL monkey patch to ensure tests - # run smoothly with standard u1db. - test_https.TestHttpSyncTargetHttpsSupport.setUp(self) - # so here monkey patch again to test our functionality. - api = client.api - http_client._VerifiedHTTPSConnection = api.VerifiedHTTPSConnection - client.api.SOLEDAD_CERT = http_client.CA_CERTS - - def test_cannot_verify_cert(self): - self.startServer() - # don't print expected traceback server-side - self.server.handle_error = lambda req, cli_addr: None - self.request_state._create_database('test') - remote_target = self.getSyncTarget( - 'localhost', 'test', cert_file=http_client.CA_CERTS) - d = remote_target.record_sync_info('other-id', 2, 'T-id') - - def _assert_raises(result): - from twisted.python.failure import Failure - if isinstance(result, Failure): - from OpenSSL.SSL import Error - error = result.value.message[0].value - if isinstance(error, Error): - msg = error.message[0][2] - self.assertEqual("certificate verify failed", msg) - return - self.fail("certificate verification should have failed.") - - d.addCallbacks(_assert_raises, _assert_raises) - return d - - def test_working(self): - """ - Test that SSL connections work well. - - This test was adapted to patch Soledad's HTTPS connection custom class - with the intended CA certificates. - """ - self.startServer() - db = self.request_state._create_database('test') - remote_target = self.getSyncTarget('localhost', 'test') - d = remote_target.record_sync_info('other-id', 2, 'T-id') - d.addCallback(lambda _: - self.assertEqual( - (2, 'T-id'), - db._get_replica_gen_and_trans_id('other-id') - )) - d.addCallback(lambda _: remote_target.close()) - return d - - def test_host_mismatch(self): - """ - This test is disabled because soledad's twisted-based http agent uses - pyOpenSSL, which will complain if we try to use an IP to connect to - the remote host (see the original test in u1db_tests/test_https.py). - """ - pass diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py deleted file mode 100644 index 20fe8579..00000000 --- a/common/src/leap/soledad/common/tests/test_server.py +++ /dev/null @@ -1,664 +0,0 @@ -# -*- coding: utf-8 -*- -# test_server.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Tests for server-related functionality. -""" -import os -import tempfile -import mock -import time -import binascii -from pkg_resources import resource_filename -from uuid import uuid4 -from hashlib import sha512 - -from urlparse import urljoin -from twisted.internet import defer -from twisted.trial import unittest - -from leap.soledad.common.couch.state import CouchServerState -from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer -from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.util import ( - make_token_soledad_app, - make_soledad_document_for_test, - soledad_sync_target, - BaseSoledadTest, -) - -from leap.soledad.common import crypto -from leap.soledad.client import Soledad -from leap.soledad.server import LockResource -from leap.soledad.server import load_configuration -from leap.soledad.server import CONFIG_DEFAULTS -from leap.soledad.server.auth import URLToAuthorization -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -class ServerAuthenticationMiddlewareTestCase(CouchDBTestCase): - - def setUp(self): - super(ServerAuthenticationMiddlewareTestCase, self).setUp() - app = mock.Mock() - self._state = CouchServerState(self.couch_url) - app.state = self._state - self.auth_middleware = SoledadTokenAuthMiddleware(app) - self._authorize('valid-uuid', 'valid-token') - - def _authorize(self, uuid, token): - token_doc = {} - token_doc['_id'] = sha512(token).hexdigest() - token_doc[self._state.TOKENS_USER_ID_KEY] = uuid - token_doc[self._state.TOKENS_TYPE_KEY] = \ - self._state.TOKENS_TYPE_DEF - dbname = self._state._tokens_dbname() - db = self.couch_server.create(dbname) - db.save(token_doc) - self.addCleanup(self.delete_db, db.name) - - def test_authorized_user(self): - is_authorized = self.auth_middleware._verify_authentication_data - self.assertTrue(is_authorized('valid-uuid', 'valid-token')) - self.assertFalse(is_authorized('valid-uuid', 'invalid-token')) - self.assertFalse(is_authorized('invalid-uuid', 'valid-token')) - self.assertFalse(is_authorized('eve', 'invalid-token')) - - -class ServerAuthorizationTestCase(BaseSoledadTest): - - """ - Tests related to Soledad server authorization. - """ - - def setUp(self): - pass - - def tearDown(self): - pass - - def _make_environ(self, path_info, request_method): - return { - 'PATH_INFO': path_info, - 'REQUEST_METHOD': request_method, - } - - def test_verify_action_with_correct_dbnames(self): - """ - Test encrypting and decrypting documents. - - The following table lists the authorized actions among all possible - u1db remote actions: - - URL path | Authorized actions - -------------------------------------------------- - / | GET - /shared-db | GET - /shared-db/docs | - - /shared-db/doc/{id} | GET, PUT, DELETE - /shared-db/sync-from/{source} | - - /user-db | GET, PUT, DELETE - /user-db/docs | - - /user-db/doc/{id} | - - /user-db/sync-from/{source} | GET, PUT, POST - """ - uuid = uuid4().hex - authmap = URLToAuthorization(uuid,) - dbname = authmap._user_db_name - # test global auth - self.assertTrue( - authmap.is_authorized(self._make_environ('/', 'GET'))) - # test shared-db database resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared', 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared', 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared', 'POST'))) - # test shared-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'POST'))) - # test shared-db doc resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'PUT'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'POST'))) - # test shared-db sync resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'POST'))) - # test user-db database resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'PUT'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'POST'))) - # test user-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'POST'))) - # test user-db doc resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'POST'))) - # test user-db sync resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) - - def test_verify_action_with_wrong_dbnames(self): - """ - Test if authorization fails for a wrong dbname. - """ - uuid = uuid4().hex - authmap = URLToAuthorization(uuid) - dbname = 'somedb' - # test wrong-db database resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'POST'))) - # test wrong-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'POST'))) - # test wrong-db doc resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'POST'))) - # test wrong-db sync resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) - - -class EncryptedSyncTestCase( - CouchDBTestCase, TestCaseWithServer): - - """ - Tests for encrypted sync using Soledad server backed by a couch database. - """ - - # increase twisted.trial's timeout because large files syncing might take - # some time to finish. - timeout = 500 - - @staticmethod - def make_app_with_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def _soledad_instance(self, user=None, passphrase=u'123', - prefix='', - secrets_path='secrets.json', - local_db_path='soledad.u1db', - server_url='', - cert_file=None, auth_token=None): - """ - Instantiate Soledad. - """ - - # this callback ensures we save a document which is sent to the shared - # db. - def _put_doc_side_effect(doc): - self._doc_put = doc - - if not server_url: - # attempt to find the soledad server url - server_address = None - server = getattr(self, 'server', None) - if server: - server_address = getattr(self.server, 'server_address', None) - else: - host = self.port.getHost() - server_address = (host.host, host.port) - if server_address: - server_url = 'http://%s:%d' % (server_address) - - return Soledad( - user, - passphrase, - secrets_path=os.path.join(self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, - cert_file=cert_file, - auth_token=auth_token, - shared_db=self.get_default_shared_mock(_put_doc_side_effect)) - - def make_app(self): - self.request_state = CouchServerState(self.couch_url) - return self.make_app_with_state(self.request_state) - - def setUp(self): - # the order of the following initializations is crucial because of - # dependencies. - # XXX explain better - CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - TestCaseWithServer.setUp(self) - - def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - - def _test_encrypted_sym_sync(self, passphrase=u'123', doc_size=2, - number_of_docs=1): - """ - Test the complete syncing chain between two soledad dbs using a - Soledad server backed by a couch database. - """ - self.startTwistedServer() - user = 'user-' + uuid4().hex - - # instantiate soledad and create a document - sol1 = self._soledad_instance( - user=user, - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token', - passphrase=passphrase) - - # instantiate another soledad using the same secret as the previous - # one (so we can correctly verify the mac of the synced document) - sol2 = self._soledad_instance( - user=user, - prefix='x', - auth_token='auth-token', - secrets_path=sol1._secrets_path, - passphrase=passphrase) - - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self.couch_url, 'user-' + user), - create=True, - ensure_ddocs=True) - - def _db1AssertEmptyDocList(results): - _, doclist = results - self.assertEqual([], doclist) - - def _db1CreateDocs(results): - deferreds = [] - for i in xrange(number_of_docs): - content = binascii.hexlify(os.urandom(doc_size / 2)) - deferreds.append(sol1.create_doc({'data': content})) - return defer.DeferredList(deferreds) - - def _db1AssertDocsSyncedToServer(results): - _, sol_doclist = results - self.assertEqual(number_of_docs, len(sol_doclist)) - # assert doc was sent to couch db - _, couch_doclist = db.get_all_docs() - self.assertEqual(number_of_docs, len(couch_doclist)) - for i in xrange(number_of_docs): - soldoc = sol_doclist.pop() - couchdoc = couch_doclist.pop() - # assert document structure in couch server - self.assertEqual(soldoc.doc_id, couchdoc.doc_id) - self.assertEqual(soldoc.rev, couchdoc.rev) - self.assertEqual(6, len(couchdoc.content)) - self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content) - - d = sol1.get_all_docs() - d.addCallback(_db1AssertEmptyDocList) - d.addCallback(_db1CreateDocs) - d.addCallback(lambda _: sol1.sync()) - d.addCallback(lambda _: sol1.get_all_docs()) - d.addCallback(_db1AssertDocsSyncedToServer) - - def _db2AssertEmptyDocList(results): - _, doclist = results - self.assertEqual([], doclist) - - def _getAllDocsFromBothDbs(results): - d1 = sol1.get_all_docs() - d2 = sol2.get_all_docs() - return defer.DeferredList([d1, d2]) - - d.addCallback(lambda _: sol2.get_all_docs()) - d.addCallback(_db2AssertEmptyDocList) - d.addCallback(lambda _: sol2.sync()) - d.addCallback(_getAllDocsFromBothDbs) - - def _assertDocSyncedFromDb1ToDb2(results): - r1, r2 = results - _, (gen1, doclist1) = r1 - _, (gen2, doclist2) = r2 - self.assertEqual(number_of_docs, gen1) - self.assertEqual(number_of_docs, gen2) - self.assertEqual(number_of_docs, len(doclist1)) - self.assertEqual(number_of_docs, len(doclist2)) - self.assertEqual(doclist1[0], doclist2[0]) - - d.addCallback(_assertDocSyncedFromDb1ToDb2) - - def _cleanUp(results): - db.delete_database() - db.close() - sol1.close() - sol2.close() - - d.addCallback(_cleanUp) - - return d - - def test_encrypted_sym_sync(self): - return self._test_encrypted_sym_sync() - - def test_encrypted_sym_sync_with_unicode_passphrase(self): - """ - Test the complete syncing chain between two soledad dbs using a - Soledad server backed by a couch database, using an unicode - passphrase. - """ - return self._test_encrypted_sym_sync(passphrase=u'ãáà äéà ëÃìïóòöõúùüñç') - - def test_sync_very_large_files(self): - """ - Test if Soledad can sync very large files. - """ - self.skipTest( - "Work in progress. For reference, see: " - "https://leap.se/code/issues/7370") - length = 100 * (10 ** 6) # 100 MB - return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1) - - def test_sync_many_small_files(self): - """ - Test if Soledad can sync many smallfiles. - """ - return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100) - - -class LockResourceTestCase( - CouchDBTestCase, TestCaseWithServer): - - """ - Tests for use of PUT and DELETE on lock resource. - """ - - @staticmethod - def make_app_with_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def setUp(self): - # the order of the following initializations is crucial because of - # dependencies. - # XXX explain better - CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - TestCaseWithServer.setUp(self) - # create the databases - db = CouchDatabase.open_database( - urljoin(self.couch_url, ('shared-%s' % (uuid4().hex))), - create=True, - ensure_ddocs=True) - self.addCleanup(db.delete_database) - self._state = CouchServerState(self.couch_url) - self._state.open_database = mock.Mock(return_value=db) - - def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - - def test__try_obtain_filesystem_lock(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - self.assertFalse(lr._lock.locked) - self.assertTrue(lr._try_obtain_filesystem_lock()) - self.assertTrue(lr._lock.locked) - lr._try_release_filesystem_lock() - - def test__try_release_filesystem_lock(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - lr._try_obtain_filesystem_lock() - self.assertTrue(lr._lock.locked) - lr._try_release_filesystem_lock() - self.assertFalse(lr._lock.locked) - - def test_put(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # lock! - lr.put({}, None) - # assert lock document was correctly written - lock_doc = lr._shared_db.get_doc('lock-' + lock_uuid) - self.assertIsNotNone(lock_doc) - self.assertTrue(LockResource.TIMESTAMP_KEY in lock_doc.content) - self.assertTrue(LockResource.LOCK_TOKEN_KEY in lock_doc.content) - timestamp = lock_doc.content[LockResource.TIMESTAMP_KEY] - token = lock_doc.content[LockResource.LOCK_TOKEN_KEY] - self.assertTrue(timestamp < time.time()) - self.assertTrue(time.time() < timestamp + LockResource.TIMEOUT) - # assert response to user - responder.send_response_json.assert_called_with( - 201, token=token, - timeout=LockResource.TIMEOUT) - - def test_delete(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # lock! - lr.put({}, None) - lock_doc = lr._shared_db.get_doc('lock-' + lock_uuid) - token = lock_doc.content[LockResource.LOCK_TOKEN_KEY] - # unlock! - lr.delete({'token': token}, None) - self.assertFalse(lr._lock.locked) - self.assertIsNone(lr._shared_db.get_doc('lock-' + lock_uuid)) - responder.send_response_json.assert_called_with(200) - - def test_put_while_locked_fails(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # lock! - lr.put({}, None) - # try to lock again! - lr.put({}, None) - self.assertEqual( - len(responder.send_response_json.call_args), 2) - self.assertEqual( - responder.send_response_json.call_args[0], (403,)) - self.assertEqual( - len(responder.send_response_json.call_args[1]), 2) - self.assertTrue( - 'remaining' in responder.send_response_json.call_args[1]) - self.assertTrue( - responder.send_response_json.call_args[1]['remaining'] > 0) - - def test_unlock_unexisting_lock_fails(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # unlock! - lr.delete({'token': 'anything'}, None) - responder.send_response_json.assert_called_with( - 404, error='lock not found') - - def test_unlock_with_wrong_token_fails(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # lock! - lr.put({}, None) - # unlock! - lr.delete({'token': 'wrongtoken'}, None) - self.assertIsNotNone(lr._shared_db.get_doc('lock-' + lock_uuid)) - responder.send_response_json.assert_called_with( - 401, error='unlock unauthorized') - - -class ConfigurationParsingTest(unittest.TestCase): - - def setUp(self): - self.maxDiff = None - - def test_use_defaults_on_failure(self): - config = load_configuration('this file will never exist') - expected = CONFIG_DEFAULTS - self.assertEquals(expected, config) - - def test_security_values_configuration(self): - # given - config_path = resource_filename('leap.soledad.common.tests', - 'fixture_soledad.conf') - # when - config = load_configuration(config_path) - - # then - expected = {'members': ['user1', 'user2'], - 'members_roles': ['role1', 'role2'], - 'admins': ['user3', 'user4'], - 'admins_roles': ['role3', 'role3']} - self.assertDictEqual(expected, config['database-security']) - - def test_server_values_configuration(self): - # given - config_path = resource_filename('leap.soledad.common.tests', - 'fixture_soledad.conf') - # when - config = load_configuration(config_path) - - # then - expected = {'couch_url': - 'http://soledad:passwd@localhost:5984', - 'create_cmd': - 'sudo -u soledad-admin /usr/bin/create-user-db', - 'admin_netrc': - '/etc/couchdb/couchdb-soledad-admin.netrc', - 'batching': False} - self.assertDictEqual(expected, config['soledad-server']) diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py deleted file mode 100644 index 36c4003c..00000000 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ /dev/null @@ -1,372 +0,0 @@ -# -*- coding: utf-8 -*- -# test_soledad.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Tests for general Soledad functionality. -""" -import os - -from mock import Mock - -from twisted.internet import defer - -from leap.common.events import catalog -from leap.soledad.common.tests.util import ( - BaseSoledadTest, - ADDRESS, -) -from leap import soledad -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.errors import DatabaseAccessError -from leap.soledad.client import Soledad -from leap.soledad.client.adbapi import U1DBConnectionPool -from leap.soledad.client.secrets import PassphraseTooShort -from leap.soledad.client.shared_db import SoledadSharedDatabase - - -class AuxMethodsTestCase(BaseSoledadTest): - - def test__init_dirs(self): - sol = self._soledad_instance(prefix='_init_dirs') - local_db_dir = os.path.dirname(sol.local_db_path) - secrets_path = os.path.dirname(sol.secrets.secrets_path) - self.assertTrue(os.path.isdir(local_db_dir)) - self.assertTrue(os.path.isdir(secrets_path)) - - def _close_soledad(results): - sol.close() - - d = sol.create_doc({}) - d.addCallback(_close_soledad) - return d - - def test__init_u1db_sqlcipher_backend(self): - sol = self._soledad_instance(prefix='_init_db') - self.assertIsInstance(sol._dbpool, U1DBConnectionPool) - self.assertTrue(os.path.isfile(sol.local_db_path)) - sol.close() - - def test__init_config_with_defaults(self): - """ - Test if configuration defaults point to the correct place. - """ - - class SoledadMock(Soledad): - - def __init__(self): - pass - - # instantiate without initializing so we just test - # _init_config_with_defaults() - sol = SoledadMock() - sol._passphrase = u'' - sol._server_url = '' - sol._init_config_with_defaults() - # assert value of local_db_path - self.assertEquals( - os.path.join(sol.default_prefix, 'soledad.u1db'), - sol.local_db_path) - - def test__init_config_from_params(self): - """ - Test if configuration is correctly read from file. - """ - sol = self._soledad_instance( - 'leap@leap.se', - passphrase=u'123', - secrets_path='value_3', - local_db_path='value_2', - server_url='value_1', - cert_file=None) - self.assertEqual( - os.path.join(self.tempdir, 'value_3'), - sol.secrets.secrets_path) - self.assertEqual( - os.path.join(self.tempdir, 'value_2'), - sol.local_db_path) - self.assertEqual('value_1', sol._server_url) - sol.close() - - def test_change_passphrase(self): - """ - Test if passphrase can be changed. - """ - prefix = '_change_passphrase' - sol = self._soledad_instance( - 'leap@leap.se', - passphrase=u'123', - prefix=prefix, - ) - - def _change_passphrase(doc1): - self._doc1 = doc1 - sol.change_passphrase(u'654321') - sol.close() - - def _assert_wrong_password_raises(results): - with self.assertRaises(DatabaseAccessError): - self._soledad_instance( - 'leap@leap.se', - passphrase=u'123', - prefix=prefix) - - def _instantiate_with_new_passphrase(results): - sol2 = self._soledad_instance( - 'leap@leap.se', - passphrase=u'654321', - prefix=prefix) - self._sol2 = sol2 - return sol2.get_doc(self._doc1.doc_id) - - def _assert_docs_are_equal(doc2): - self.assertEqual(self._doc1, doc2) - self._sol2.close() - - d = sol.create_doc({'simple': 'doc'}) - d.addCallback(_change_passphrase) - d.addCallback(_assert_wrong_password_raises) - d.addCallback(_instantiate_with_new_passphrase) - d.addCallback(_assert_docs_are_equal) - d.addCallback(lambda _: sol.close()) - - return d - - def test_change_passphrase_with_short_passphrase_raises(self): - """ - Test if attempt to change passphrase passing a short passphrase - raises. - """ - sol = self._soledad_instance( - 'leap@leap.se', - passphrase=u'123') - # check that soledad complains about new passphrase length - self.assertRaises( - PassphraseTooShort, - sol.change_passphrase, u'54321') - sol.close() - - def test_get_passphrase(self): - """ - Assert passphrase getter works fine. - """ - sol = self._soledad_instance() - self.assertEqual('123', sol._passphrase) - sol.close() - - -class SoledadSharedDBTestCase(BaseSoledadTest): - - """ - These tests ensure the functionalities of the shared recovery database. - """ - - def setUp(self): - BaseSoledadTest.setUp(self) - self._shared_db = SoledadSharedDatabase( - 'https://provider/', ADDRESS, document_factory=SoledadDocument, - creds=None) - - def tearDown(self): - BaseSoledadTest.tearDown(self) - - def test__get_secrets_from_shared_db(self): - """ - Ensure the shared db is queried with the correct doc_id. - """ - doc_id = self._soledad.secrets._shared_db_doc_id() - self._soledad.secrets._get_secrets_from_shared_db() - self.assertTrue( - self._soledad.shared_db.get_doc.assert_called_with( - doc_id) is None, - 'Wrong doc_id when fetching recovery document.') - - def test__put_secrets_in_shared_db(self): - """ - Ensure recovery document is put into shared recover db. - """ - doc_id = self._soledad.secrets._shared_db_doc_id() - self._soledad.secrets._put_secrets_in_shared_db() - self.assertTrue( - self._soledad.shared_db.get_doc.assert_called_with( - doc_id) is None, - 'Wrong doc_id when fetching recovery document.') - self.assertTrue( - self._soledad.shared_db.put_doc.assert_called_with( - self._doc_put) is None, - 'Wrong document when putting recovery document.') - self.assertTrue( - self._doc_put.doc_id == doc_id, - 'Wrong doc_id when putting recovery document.') - - -class SoledadSignalingTestCase(BaseSoledadTest): - - """ - These tests ensure signals are correctly emmited by Soledad. - """ - - EVENTS_SERVER_PORT = 8090 - - def setUp(self): - # mock signaling - soledad.client.signal = Mock() - soledad.client.secrets.events.emit_async = Mock() - # run parent's setUp - BaseSoledadTest.setUp(self) - - def tearDown(self): - BaseSoledadTest.tearDown(self) - - def _pop_mock_call(self, mocked): - mocked.call_args_list.pop() - mocked.mock_calls.pop() - mocked.call_args = mocked.call_args_list[-1] - - def test_stage3_bootstrap_signals(self): - """ - Test that a fresh soledad emits all bootstrap signals. - - Signals are: - - downloading keys / done downloading keys. - - creating keys / done creating keys. - - downloading keys / done downloading keys. - - uploading keys / done uploading keys. - """ - soledad.client.secrets.events.emit_async.reset_mock() - # get a fresh instance so it emits all bootstrap signals - sol = self._soledad_instance( - secrets_path='alternative_stage3.json', - local_db_path='alternative_stage3.u1db', - userid=ADDRESS) - # reverse call order so we can verify in the order the signals were - # expected - soledad.client.secrets.events.emit_async.mock_calls.reverse() - soledad.client.secrets.events.emit_async.call_args = \ - soledad.client.secrets.events.emit_async.call_args_list[0] - soledad.client.secrets.events.emit_async.call_args_list.reverse() - - user_data = {'userid': ADDRESS, 'uuid': ADDRESS} - - # downloading keys signals - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data - ) - # creating keys signals - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_CREATING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_CREATING_KEYS, user_data - ) - # downloading once more (inside _put_keys_in_shared_db) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data - ) - # uploading keys signals - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_UPLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data - ) - # assert db was locked and unlocked - sol.shared_db.lock.assert_called_with() - sol.shared_db.unlock.assert_called_with('atoken') - sol.close() - - def test_stage2_bootstrap_signals(self): - """ - Test that if there are keys in server, soledad will download them and - emit corresponding signals. - """ - # get existing instance so we have access to keys - sol = self._soledad_instance() - # create a document with secrets - doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id()) - doc.content = sol.secrets._export_recovery_document() - sol.close() - # reset mock - soledad.client.secrets.events.emit_async.reset_mock() - # get a fresh instance so it emits all bootstrap signals - shared_db = self.get_default_shared_mock(get_doc_return_value=doc) - sol = self._soledad_instance( - secrets_path='alternative_stage2.json', - local_db_path='alternative_stage2.u1db', - shared_db_class=shared_db) - # reverse call order so we can verify in the order the signals were - # expected - soledad.client.secrets.events.emit_async.mock_calls.reverse() - soledad.client.secrets.events.emit_async.call_args = \ - soledad.client.secrets.events.emit_async.call_args_list[0] - soledad.client.secrets.events.emit_async.call_args_list.reverse() - # assert download keys signals - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, - {'userid': ADDRESS, 'uuid': ADDRESS} - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, - {'userid': ADDRESS, 'uuid': ADDRESS}, - ) - sol.close() - - def test_stage1_bootstrap_signals(self): - """ - Test that if soledad already has a local secret, it emits no signals. - """ - soledad.client.signal.reset_mock() - # get an existent instance so it emits only some of bootstrap signals - sol = self._soledad_instance() - self.assertEqual([], soledad.client.signal.mock_calls) - sol.close() - - @defer.inlineCallbacks - def test_sync_signals(self): - """ - Test Soledad emits SOLEDAD_CREATING_KEYS signal. - """ - # get a fresh instance so it emits all bootstrap signals - sol = self._soledad_instance() - soledad.client.signal.reset_mock() - - # mock the actual db sync so soledad does not try to connect to the - # server - d = defer.Deferred() - d.callback(None) - sol._dbsyncer.sync = Mock(return_value=d) - - yield sol.sync() - - # assert the signal has been emitted - soledad.client.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DATA_SYNC, - {'userid': ADDRESS, 'uuid': ADDRESS}, - ) - sol.close() diff --git a/common/src/leap/soledad/common/tests/test_soledad_app.py b/common/src/leap/soledad/common/tests/test_soledad_app.py deleted file mode 100644 index 4598a7bb..00000000 --- a/common/src/leap/soledad/common/tests/test_soledad_app.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -# test_soledad_app.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Test ObjectStore and Couch backend bits. -""" - - -from testscenarios import TestWithScenarios - -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.tests.util import make_soledad_document_for_test -from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.tests.util import make_token_soledad_app -from leap.soledad.common.tests.util import make_token_http_database_for_test -from leap.soledad.common.tests.util import copy_token_http_database_for_test -from leap.soledad.common.tests.u1db_tests import test_backends - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -# ----------------------------------------------------------------------------- - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_soledad_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -class SoledadTests( - TestWithScenarios, test_backends.AllDatabaseTests, BaseSoledadTest): - - scenarios = LEAP_SCENARIOS + [ - ('token_http', { - 'make_database_for_test': make_token_http_database_for_test, - 'copy_database_for_test': copy_token_http_database_for_test, - 'make_document_for_test': make_soledad_document_for_test, - 'make_app_with_state': make_token_soledad_app, - }) - ] diff --git a/common/src/leap/soledad/common/tests/test_soledad_doc.py b/common/src/leap/soledad/common/tests/test_soledad_doc.py deleted file mode 100644 index df9fd09e..00000000 --- a/common/src/leap/soledad/common/tests/test_soledad_doc.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -# test_soledad_doc.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test Leap backend bits: soledad docs -""" -from testscenarios import TestWithScenarios - -from leap.soledad.common.tests.u1db_tests import test_document -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.tests.util import make_soledad_document_for_test - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_document`. -# ----------------------------------------------------------------------------- - -class TestSoledadDocument( - TestWithScenarios, - test_document.TestDocument, BaseSoledadTest): - - scenarios = ([( - 'leap', { - 'make_document_for_test': make_soledad_document_for_test})]) - - -class TestSoledadPyDocument( - TestWithScenarios, - test_document.TestPyDocument, BaseSoledadTest): - - scenarios = ([( - 'leap', { - 'make_document_for_test': make_soledad_document_for_test})]) diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py deleted file mode 100644 index 8105c56e..00000000 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ /dev/null @@ -1,721 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sqlcipher.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test sqlcipher backend internals. -""" -import os -import time -import threading -import tempfile -import shutil - -from pysqlcipher import dbapi2 -from testscenarios import TestWithScenarios - -# u1db stuff. -from u1db import errors -from u1db import query_parser -from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase - -# soledad stuff. -from leap.soledad.common import soledad_assert -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import DatabaseIsNotEncrypted - -# u1db tests stuff. -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import test_open -from leap.soledad.common.tests.util import make_sqlcipher_database_for_test -from leap.soledad.common.tests.util import copy_sqlcipher_database_for_test -from leap.soledad.common.tests.util import PASSWORD -from leap.soledad.common.tests.util import BaseSoledadTest - - -def sqlcipher_open(path, passphrase, create=True, document_factory=None): - return SQLCipherDatabase( - SQLCipherOptions(path, passphrase, create=create)) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_common_backend`. -# ----------------------------------------------------------------------------- - -class TestSQLCipherBackendImpl(tests.TestCase): - - def test__allocate_doc_id(self): - db = sqlcipher_open(':memory:', PASSWORD) - doc_id1 = db._allocate_doc_id() - self.assertTrue(doc_id1.startswith('D-')) - self.assertEqual(34, len(doc_id1)) - int(doc_id1[len('D-'):], 16) - self.assertNotEqual(doc_id1, db._allocate_doc_id()) - db.close() - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -# ----------------------------------------------------------------------------- - -def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): - return SoledadDocument(doc_id, rev, content, has_conflicts=has_conflicts) - - -SQLCIPHER_SCENARIOS = [ - ('sqlcipher', {'make_database_for_test': make_sqlcipher_database_for_test, - 'copy_database_for_test': copy_sqlcipher_database_for_test, - 'make_document_for_test': make_document_for_test, }), -] - - -class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherDatabaseTests(TestWithScenarios, - test_backends.LocalDatabaseTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherValidateGenNTransIdTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateGenNTransIdTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherValidateSourceGenTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateSourceGenTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherWithConflictsTests( - TestWithScenarios, - test_backends.LocalDatabaseWithConflictsTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherIndexTests( - TestWithScenarios, test_backends.DatabaseIndexTests): - scenarios = SQLCIPHER_SCENARIOS - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sqlite_backend`. -# ----------------------------------------------------------------------------- - -class TestSQLCipherDatabase(tests.TestCase): - """ - Tests from u1db.tests.test_sqlite_backend.TestSQLiteDatabase. - """ - - def test_atomic_initialize(self): - # This test was modified to ensure that db2.close() is called within - # the thread that created the database. - tmpdir = self.createTempDir() - dbname = os.path.join(tmpdir, 'atomic.db') - - t2 = None # will be a thread - - class SQLCipherDatabaseTesting(SQLCipherDatabase): - _index_storage_value = "testing" - - def __init__(self, dbname, ntry): - self._try = ntry - self._is_initialized_invocations = 0 - SQLCipherDatabase.__init__( - self, - SQLCipherOptions(dbname, PASSWORD)) - - def _is_initialized(self, c): - res = \ - SQLCipherDatabase._is_initialized(self, c) - if self._try == 1: - self._is_initialized_invocations += 1 - if self._is_initialized_invocations == 2: - t2.start() - # hard to do better and have a generic test - time.sleep(0.05) - return res - - class SecondTry(threading.Thread): - - outcome2 = [] - - def run(self): - try: - db2 = SQLCipherDatabaseTesting(dbname, 2) - except Exception, e: - SecondTry.outcome2.append(e) - else: - SecondTry.outcome2.append(db2) - - t2 = SecondTry() - db1 = SQLCipherDatabaseTesting(dbname, 1) - t2.join() - - self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) - self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) - db1.close() - - -class TestAlternativeDocument(SoledadDocument): - - """A (not very) alternative implementation of Document.""" - - -class TestSQLCipherPartialExpandDatabase(tests.TestCase): - """ - Tests from u1db.tests.test_sqlite_backend.TestSQLitePartialExpandDatabase. - """ - - # The following tests had to be cloned from u1db because they all - # instantiate the backend directly, so we need to change that in order to - # our backend be instantiated in place. - - def setUp(self): - self.db = sqlcipher_open(':memory:', PASSWORD) - - def tearDown(self): - self.db.close() - - def test_default_replica_uid(self): - self.assertIsNot(None, self.db._replica_uid) - self.assertEqual(32, len(self.db._replica_uid)) - int(self.db._replica_uid, 16) - - def test__parse_index(self): - g = self.db._parse_index_definition('fieldname') - self.assertIsInstance(g, query_parser.ExtractField) - self.assertEqual(['fieldname'], g.field) - - def test__update_indexes(self): - g = self.db._parse_index_definition('fieldname') - c = self.db._get_sqlite_handle().cursor() - self.db._update_indexes('doc-id', {'fieldname': 'val'}, - [('fieldname', g)], c) - c.execute('SELECT doc_id, field_name, value FROM document_fields') - self.assertEqual([('doc-id', 'fieldname', 'val')], - c.fetchall()) - - def test_create_database(self): - raw_db = self.db._get_sqlite_handle() - self.assertNotEqual(None, raw_db) - - def test__set_replica_uid(self): - # Start from scratch, so that replica_uid isn't set. - self.assertIsNot(None, self.db._real_replica_uid) - self.assertIsNot(None, self.db._replica_uid) - self.db._set_replica_uid('foo') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT value FROM u1db_config WHERE name='replica_uid'") - self.assertEqual(('foo',), c.fetchone()) - self.assertEqual('foo', self.db._real_replica_uid) - self.assertEqual('foo', self.db._replica_uid) - self.db._close_sqlite_handle() - self.assertEqual('foo', self.db._replica_uid) - - def test__open_database(self): - # SQLCipherDatabase has no _open_database() method, so we just pass - # (and test for the same funcionality on test_open_database_existing() - # below). - pass - - def test__open_database_with_factory(self): - # SQLCipherDatabase has no _open_database() method. - pass - - def test__open_database_non_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, - path, PASSWORD, create=False) - - def test__open_database_during_init(self): - # The purpose of this test is to ensure that _open_database() parallel - # db initialization behaviour is correct. As SQLCipherDatabase does - # not have an _open_database() method, we just do not implement this - # test. - pass - - def test__open_database_invalid(self): - # This test was modified to ensure that an empty database file will - # raise a DatabaseIsNotEncrypted exception instead of a - # dbapi2.OperationalError exception. - temp_dir = self.createTempDir(prefix='u1db-test-') - path1 = temp_dir + '/invalid1.db' - with open(path1, 'wb') as f: - f.write("") - self.assertRaises(DatabaseIsNotEncrypted, - sqlcipher_open, path1, - PASSWORD) - with open(path1, 'wb') as f: - f.write("invalid") - self.assertRaises(dbapi2.DatabaseError, - sqlcipher_open, path1, - PASSWORD) - - def test_open_database_existing(self): - # In the context of SQLCipherDatabase, where no _open_database() - # method exists and thus there's no call to _which_index_storage(), - # this test tests for the same functionality as - # test_open_database_create() below. So, we just pass. - pass - - def test_open_database_with_factory(self): - # SQLCipherDatabase's constructor has no factory parameter. - pass - - def test_open_database_create(self): - # SQLCipherDatabas has no open_database() method, so we just test for - # the actual database constructor effects. - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/new.sqlite' - db1 = sqlcipher_open(path, PASSWORD, create=True) - db2 = sqlcipher_open(path, PASSWORD, create=False) - self.assertIsInstance(db2, SQLCipherDatabase) - db1.close() - db2.close() - - def test_create_database_initializes_schema(self): - # This test had to be cloned because our implementation of SQLCipher - # backend is referenced with an index_storage_value that includes the - # word "encrypted". See u1db's sqlite_backend and our - # sqlcipher_backend for reference. - raw_db = self.db._get_sqlite_handle() - c = raw_db.cursor() - c.execute("SELECT * FROM u1db_config") - config = dict([(r[0], r[1]) for r in c.fetchall()]) - replica_uid = self.db._replica_uid - self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, - 'index_storage': 'expand referenced encrypted'}, - config) - - def test_store_syncable(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - # assert that docs are syncable by default - self.assertEqual(True, doc.syncable) - # assert that we can store syncable = False - doc.syncable = False - self.db.put_doc(doc) - self.assertEqual(False, self.db.get_doc(doc.doc_id).syncable) - # assert that we can store syncable = True - doc.syncable = True - self.db.put_doc(doc) - self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable) - - def test__close_sqlite_handle(self): - raw_db = self.db._get_sqlite_handle() - self.db._close_sqlite_handle() - self.assertRaises(dbapi2.ProgrammingError, - raw_db.cursor) - - def test__get_generation(self): - self.assertEqual(0, self.db._get_generation()) - - def test__get_generation_info(self): - self.assertEqual((0, ''), self.db._get_generation_info()) - - def test_create_index(self): - self.db.create_index('test-idx', "key") - self.assertEqual([('test-idx', ["key"])], self.db.list_indexes()) - - def test_create_index_multiple_fields(self): - self.db.create_index('test-idx', "key", "key2") - self.assertEqual([('test-idx', ["key", "key2"])], - self.db.list_indexes()) - - def test__get_index_definition(self): - self.db.create_index('test-idx', "key", "key2") - # TODO: How would you test that an index is getting used for an SQL - # request? - self.assertEqual(["key", "key2"], - self.db._get_index_definition('test-idx')) - - def test_list_index_mixed(self): - # Make sure that we properly order the output - c = self.db._get_sqlite_handle().cursor() - # We intentionally insert the data in weird ordering, to make sure the - # query still gets it back correctly. - c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", - [('idx-1', 0, 'key10'), - ('idx-2', 2, 'key22'), - ('idx-1', 1, 'key11'), - ('idx-2', 0, 'key20'), - ('idx-2', 1, 'key21')]) - self.assertEqual([('idx-1', ['key10', 'key11']), - ('idx-2', ['key20', 'key21', 'key22'])], - self.db.list_indexes()) - - def test_no_indexes_no_document_fields(self): - self.db.create_doc_from_json( - '{"key1": "val1", "key2": "val2"}') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([], c.fetchall()) - - def test_create_extracts_fields(self): - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - doc2 = self.db.create_doc_from_json('{"key1": "valx", "key2": "valy"}') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([], c.fetchall()) - self.db.create_index('test', 'key1', 'key2') - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual(sorted( - [(doc1.doc_id, "key1", "val1"), - (doc1.doc_id, "key2", "val2"), - (doc2.doc_id, "key1", "valx"), - (doc2.doc_id, "key2", "valy"), ]), sorted(c.fetchall())) - - def test_put_updates_fields(self): - self.db.create_index('test', 'key1', 'key2') - doc1 = self.db.create_doc_from_json( - '{"key1": "val1", "key2": "val2"}') - doc1.content = {"key1": "val1", "key2": "valy"} - self.db.put_doc(doc1) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, "key1", "val1"), - (doc1.doc_id, "key2", "valy"), ], c.fetchall()) - - def test_put_updates_nested_fields(self): - self.db.create_index('test', 'key', 'sub.doc') - doc1 = self.db.create_doc_from_json(tests.nested_doc) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, "key", "value"), - (doc1.doc_id, "sub.doc", "underneath"), ], - c.fetchall()) - - def test__ensure_schema_rollback(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/rollback.db' - - class SQLitePartialExpandDbTesting(SQLCipherDatabase): - - def _set_replica_uid_in_transaction(self, uid): - super(SQLitePartialExpandDbTesting, - self)._set_replica_uid_in_transaction(uid) - if fail: - raise Exception() - - db = SQLitePartialExpandDbTesting.__new__(SQLitePartialExpandDbTesting) - db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed - fail = True - self.assertRaises(Exception, db._ensure_schema) - fail = False - db._initialize(db._db_handle.cursor()) - - def test_open_database_non_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, path, "123", - create=False) - - def test_delete_database_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/new.sqlite' - db = sqlcipher_open(path, "123", create=True) - db.close() - SQLCipherDatabase.delete_database(path) - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, path, "123", - create=False) - - def test_delete_database_nonexistent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - SQLCipherDatabase.delete_database, path) - - def test__get_indexed_fields(self): - self.db.create_index('idx1', 'a', 'b') - self.assertEqual(set(['a', 'b']), self.db._get_indexed_fields()) - self.db.create_index('idx2', 'b', 'c') - self.assertEqual(set(['a', 'b', 'c']), self.db._get_indexed_fields()) - - def test_indexed_fields_expanded(self): - self.db.create_index('idx1', 'key1') - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - self.assertEqual(set(['key1']), self.db._get_indexed_fields()) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) - - def test_create_index_updates_fields(self): - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - self.db.create_index('idx1', 'key1') - self.assertEqual(set(['key1']), self.db._get_indexed_fields()) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) - - def assertFormatQueryEquals(self, exp_statement, exp_args, definition, - values): - statement, args = self.db._format_query(definition, values) - self.assertEqual(exp_statement, statement) - self.assertEqual(exp_args, args) - - def test__format_query(self): - self.assertFormatQueryEquals( - "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " - "document d, document_fields d0 LEFT OUTER JOIN conflicts c ON " - "c.doc_id = d.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name " - "= ? AND d0.value = ? GROUP BY d.doc_id, d.doc_rev, d.content " - "ORDER BY d0.value;", ["key1", "a"], - ["key1"], ["a"]) - - def test__format_query2(self): - self.assertFormatQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value = ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value = ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ["key1", "a", "key2", "b", "key3", "c"], - ["key1", "key2", "key3"], ["a", "b", "c"]) - - def test__format_query_wildcard(self): - self.assertFormatQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value GLOB ? AND d.doc_id = d2.doc_id AND d2.field_name = ? ' - 'AND d2.value NOT NULL GROUP BY d.doc_id, d.doc_rev, d.content ' - 'ORDER BY d0.value, d1.value, d2.value;', - ["key1", "a", "key2", "b*", "key3"], ["key1", "key2", "key3"], - ["a", "b*", "*"]) - - def assertFormatRangeQueryEquals(self, exp_statement, exp_args, definition, - start_value, end_value): - statement, args = self.db._format_range_query( - definition, start_value, end_value) - self.assertEqual(exp_statement, statement) - self.assertEqual(exp_args, args) - - def test__format_range_query(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value >= ? AND d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c', 'key1', 'p', 'key2', 'q', - 'key3', 'r'], - ["key1", "key2", "key3"], ["a", "b", "c"], ["p", "q", "r"]) - - def test__format_range_query_no_start(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c'], - ["key1", "key2", "key3"], None, ["a", "b", "c"]) - - def test__format_range_query_no_end(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value >= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c'], - ["key1", "key2", "key3"], ["a", "b", "c"], None) - - def test__format_range_query_wildcard(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value NOT NULL AND d.doc_id = d0.doc_id AND d0.field_name = ? ' - 'AND d0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? ' - 'AND (d1.value < ? OR d1.value GLOB ?) AND d.doc_id = d2.doc_id ' - 'AND d2.field_name = ? AND d2.value NOT NULL GROUP BY d.doc_id, ' - 'd.doc_rev, d.content ORDER BY d0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'key1', 'p', 'key2', 'q', 'q*', - 'key3'], - ["key1", "key2", "key3"], ["a", "b*", "*"], ["p", "q*", "*"]) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_open`. -# ----------------------------------------------------------------------------- - - -class SQLCipherOpen(test_open.TestU1DBOpen): - - def test_open_no_create(self): - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, self.db_path, - PASSWORD, - create=False) - self.assertFalse(os.path.exists(self.db_path)) - - def test_open_create(self): - db = sqlcipher_open(self.db_path, PASSWORD, create=True) - self.addCleanup(db.close) - self.assertTrue(os.path.exists(self.db_path)) - self.assertIsInstance(db, SQLCipherDatabase) - - def test_open_with_factory(self): - db = sqlcipher_open(self.db_path, PASSWORD, create=True, - document_factory=TestAlternativeDocument) - self.addCleanup(db.close) - doc = db.create_doc({}) - self.assertTrue(isinstance(doc, SoledadDocument)) - - def test_open_existing(self): - db = sqlcipher_open(self.db_path, PASSWORD) - self.addCleanup(db.close) - doc = db.create_doc_from_json(tests.simple_doc) - # Even though create=True, we shouldn't wipe the db - db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) - self.addCleanup(db2.close) - doc2 = db2.get_doc(doc.doc_id) - self.assertEqual(doc, doc2) - - def test_open_existing_no_create(self): - db = sqlcipher_open(self.db_path, PASSWORD) - self.addCleanup(db.close) - db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) - self.addCleanup(db2.close) - self.assertIsInstance(db2, SQLCipherDatabase) - - -# ----------------------------------------------------------------------------- -# Tests for actual encryption of the database -# ----------------------------------------------------------------------------- - -class SQLCipherEncryptionTests(BaseSoledadTest): - - """ - Tests to guarantee SQLCipher is indeed encrypting data when storing. - """ - - def _delete_dbfiles(self): - for dbfile in [self.DB_FILE]: - if os.path.exists(dbfile): - os.unlink(dbfile) - - def setUp(self): - # the following come from BaseLeapTest.setUpClass, because - # twisted.trial doesn't support such class methods for setting up - # test classes. - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - # this is our own stuff - self.DB_FILE = os.path.join(self.tempdir, 'test.db') - self._delete_dbfiles() - - def tearDown(self): - self._delete_dbfiles() - # the following come from BaseLeapTest.tearDownClass, because - # twisted.trial doesn't support such class methods for tearing down - # test classes. - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check! please do not wipe my home... - # XXX needs to adapt to non-linuces - soledad_assert( - self.tempdir.startswith('/tmp/leap_tests-') or - self.tempdir.startswith('/var/folder'), - "beware! tried to remove a dir which does not " - "live in temporal folder!") - shutil.rmtree(self.tempdir) - - def test_try_to_open_encrypted_db_with_sqlite_backend(self): - """ - SQLite backend should not succeed to open SQLCipher databases. - """ - db = sqlcipher_open(self.DB_FILE, PASSWORD) - doc = db.create_doc_from_json(tests.simple_doc) - db.close() - try: - # trying to open an encrypted database with the regular u1db - # backend should raise a DatabaseError exception. - SQLitePartialExpandDatabase(self.DB_FILE, - document_factory=SoledadDocument) - raise DatabaseIsNotEncrypted() - except dbapi2.DatabaseError: - # at this point we know that the regular U1DB sqlcipher backend - # did not succeed on opening the database, so it was indeed - # encrypted. - db = sqlcipher_open(self.DB_FILE, PASSWORD) - doc = db.get_doc(doc.doc_id) - self.assertEqual(tests.simple_doc, doc.get_json(), - 'decrypted content mismatch') - db.close() - - def test_try_to_open_raw_db_with_sqlcipher_backend(self): - """ - SQLCipher backend should not succeed to open unencrypted databases. - """ - db = SQLitePartialExpandDatabase(self.DB_FILE, - document_factory=SoledadDocument) - db.create_doc_from_json(tests.simple_doc) - db.close() - try: - # trying to open the a non-encrypted database with sqlcipher - # backend should raise a DatabaseIsNotEncrypted exception. - db = sqlcipher_open(self.DB_FILE, PASSWORD) - db.close() - raise dbapi2.DatabaseError( - "SQLCipher backend should not be able to open non-encrypted " - "dbs.") - except DatabaseIsNotEncrypted: - pass diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py deleted file mode 100644 index 439fc070..00000000 --- a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py +++ /dev/null @@ -1,744 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sqlcipher.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test sqlcipher backend sync. -""" - - -import os - -from u1db import sync -from u1db import vectorclock -from u1db import errors -from uuid import uuid4 - -from testscenarios import TestWithScenarios - -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client.http_target import SoledadHTTPSyncTarget -from leap.soledad.client.crypto import decrypt_doc_dict - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.test_sqlcipher import SQLCIPHER_SCENARIOS -from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.tests.test_sync_target import \ - SoledadDatabaseSyncTargetTests -from leap.soledad.common.tests.util import soledad_sync_target -from leap.soledad.common.tests.util import BaseSoledadTest - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -# ----------------------------------------------------------------------------- - -def sync_via_synchronizer_and_soledad(test, db_source, db_target, - trace_hook=None, - trace_hook_shallow=None): - if trace_hook: - test.skipTest("full trace hook unsupported over http") - path = test._http_at[db_target] - target = SoledadHTTPSyncTarget.connect( - test.getURL(path), test._soledad._crypto) - target.set_token_credentials('user-uuid', 'auth-token') - if trace_hook_shallow: - target._set_trace_hook_shallow(trace_hook_shallow) - return sync.Synchronizer(db_source, target).sync() - - -def sync_via_synchronizer(test, db_source, db_target, - trace_hook=None, - trace_hook_shallow=None): - target = db_target.get_sync_target() - trace_hook = trace_hook or trace_hook_shallow - if trace_hook: - target._set_trace_hook(trace_hook) - return sync.Synchronizer(db_source, target).sync() - - -sync_scenarios = [] -for name, scenario in SQLCIPHER_SCENARIOS: - scenario['do_sync'] = sync_via_synchronizer - sync_scenarios.append((name, scenario)) - - -class SQLCipherDatabaseSyncTests( - TestWithScenarios, - tests.DatabaseBaseTests, - BaseSoledadTest): - - """ - Test for succesfull sync between SQLCipher and LeapBackend. - - Some of the tests in this class had to be adapted because the remote - backend always receive encrypted content, and so it can not rely on - document's content comparison to try to autoresolve conflicts. - """ - - scenarios = sync_scenarios - - def setUp(self): - self._use_tracking = {} - super(tests.DatabaseBaseTests, self).setUp() - - def create_database(self, replica_uid, sync_role=None): - if replica_uid == 'test' and sync_role is None: - # created up the chain by base class but unused - return None - db = self.create_database_for_role(replica_uid, sync_role) - if sync_role: - self._use_tracking[db] = (replica_uid, sync_role) - self.addCleanup(db.close) - return db - - def create_database_for_role(self, replica_uid, sync_role): - # hook point for reuse - return tests.DatabaseBaseTests.create_database(self, replica_uid) - - def sync(self, db_from, db_to, trace_hook=None, - trace_hook_shallow=None): - from_name, from_sync_role = self._use_tracking[db_from] - to_name, to_sync_role = self._use_tracking[db_to] - if from_sync_role not in ('source', 'both'): - raise Exception("%s marked for %s use but used as source" % - (from_name, from_sync_role)) - if to_sync_role not in ('target', 'both'): - raise Exception("%s marked for %s use but used as target" % - (to_name, to_sync_role)) - return self.do_sync(self, db_from, db_to, trace_hook, - trace_hook_shallow) - - def assertLastExchangeLog(self, db, expected): - log = getattr(db, '_last_exchange_log', None) - if log is None: - return - self.assertEqual(expected, log) - - def copy_database(self, db, sync_role=None): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - db_copy = tests.DatabaseBaseTests.copy_database(self, db) - name, orig_sync_role = self._use_tracking[db] - self._use_tracking[db_copy] = (name + '(copy)', sync_role or - orig_sync_role) - return db_copy - - def test_sync_tracks_db_generation_of_other(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertEqual( - (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [], 'last_known_gen': 0}, - 'return': - {'docs': [], 'last_gen': 0}}) - - def test_sync_autoresolves(self): - """ - Test for sync autoresolve remote. - - This test was adapted because the remote database receives encrypted - content and so it can't compare documents contents to autoresolve. - """ - # The remote database can't autoresolve conflicts based on magic - # content convergence, so we modify this test to leave the possibility - # of the remode document ending up in conflicted state. - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev1 = doc1.rev - doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev2 = doc2.rev - self.sync(self.db1, self.db2) - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - # if remote content is in conflicted state, then document revisions - # will be different. - # self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) - v = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) - - def test_sync_autoresolves_moar(self): - """ - Test for sync autoresolve local. - - This test was adapted to decrypt remote content before assert. - """ - # here we test that when a database that has a conflicted document is - # the source of a sync, and the target database has a revision of the - # conflicted document that is newer than the source database's, and - # that target's database's document's content is the same as the - # source's document's conflict's, the source's document's conflict gets - # autoresolved, and the source's document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # `-------> - # a3b2 a1b2 (autoresolved) - # `-------> - # a3b2 a3b2 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - # because the conflict is on the source, sync it another time - self.sync(self.db1, self.db2) - # make sure db2 now has the exact same thing - doc1 = self.db1.get_doc('doc') - self.assertGetEncryptedDoc( - self.db2, - doc1.doc_id, doc1.rev, doc1.get_json(), False) - - def test_sync_autoresolves_moar_backwards(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # Despite that, in Soledad we suppose that the server never syncs, so - # it never has conflicted documents. Also, if it had, convergence - # would not be possible by checking document's contents because they - # would be encrypted in server. - # - # Therefore we suppress this test. - pass - - def test_sync_autoresolves_moar_backwards_three(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # We use the same reasoning from the last test to suppress this one. - pass - - def test_sync_pulling_doesnt_update_other_if_changed(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(tests.simple_doc) - # After the local side has sent its list of docs, before we start - # receiving the "targets" response, we update the local database with a - # new record. - # When we finish synchronizing, we can notice that something locally - # was updated, and we cannot tell c2 our new updated generation - - def before_get_docs(state): - if state != 'before get_docs': - return - self.db1.create_doc_from_json(tests.simple_doc) - - self.assertEqual(0, self.sync(self.db1, self.db2, - trace_hook=before_get_docs)) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [], 'last_known_gen': 0}, - 'return': - {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - # c2 should not have gotten a '_record_sync_info' call, because the - # local database had been updated more than just by the messages - # returned from c2. - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - - def test_sync_doesnt_update_other_if_nothing_pulled(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc) - - def no_record_sync_info(state): - if state != 'record_sync_info': - return - self.fail('SyncTarget.record_sync_info was called') - self.assertEqual(1, self.sync(self.db1, self.db2, - trace_hook_shallow=no_record_sync_info)) - self.assertEqual( - 1, - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) - - def test_sync_ignores_convergence(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(tests.simple_doc) - self.db3 = self.create_database('test3', 'target') - self.assertEqual(1, self.sync(self.db1, self.db3)) - self.assertEqual(0, self.sync(self.db2, self.db3)) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_ignores_superseded(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(tests.simple_doc) - doc_rev1 = doc.rev - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.sync(self.db2, self.db3) - new_content = '{"key": "altval"}' - doc.set_json(new_content) - self.db1.put_doc(doc) - doc_rev2 = doc.rev - self.sync(self.db2, self.db1) - self.assertLastExchangeLog(self.db1, - {'receive': - {'docs': [(doc.doc_id, doc_rev1)], - 'source_uid': 'test2', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': - {'docs': [(doc.doc_id, doc_rev2)], - 'last_gen': 2}}) - self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) - - def test_sync_sees_remote_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc1.doc_id - doc1_rev = doc1.rev - self.db1.create_index('test-idx', 'key') - new_doc = '{"key": "altval"}' - doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) - doc2_rev = doc2.rev - self.assertTransactionLog([doc1.doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc_id, doc1_rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': - {'docs': [(doc_id, doc2_rev)], - 'last_gen': 1}}) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) - self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) - from_idx = self.db1.get_from_index('test-idx', 'altval')[0] - self.assertEqual(doc2.doc_id, from_idx.doc_id) - self.assertEqual(doc2.rev, from_idx.rev) - self.assertTrue(from_idx.has_conflicts) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - - def test_sync_sees_remote_delete_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc1.doc_id - self.db1.create_index('test-idx', 'key') - self.sync(self.db1, self.db2) - doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) - new_doc = '{"key": "altval"}' - doc1.set_json(new_doc) - self.db1.put_doc(doc1) - self.db2.delete_doc(doc2) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc_id, doc1.rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [(doc_id, doc2.rev)], - 'last_gen': 2}}) - self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) - self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, doc2.rev, None, False) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - - def test_sync_local_race_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc.doc_id - doc1_rev = doc.rev - self.db1.create_index('test-idx', 'key') - self.sync(self.db1, self.db2) - content1 = '{"key": "localval"}' - content2 = '{"key": "altval"}' - doc.set_json(content2) - self.db2.put_doc(doc) - doc2_rev2 = doc.rev - triggered = [] - - def after_whatschanged(state): - if state != 'after whats_changed': - return - triggered.append(True) - doc = self.make_document(doc_id, doc1_rev, content1) - self.db1.put_doc(doc) - - self.sync(self.db1, self.db2, trace_hook=after_whatschanged) - self.assertEqual([True], triggered) - self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) - from_idx = self.db1.get_from_index('test-idx', 'altval')[0] - self.assertEqual(doc.doc_id, from_idx.doc_id) - self.assertEqual(doc.rev, from_idx.rev) - self.assertTrue(from_idx.has_conflicts) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - self.assertEqual([], self.db1.get_from_index('test-idx', 'localval')) - - def test_sync_propagates_deletes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc1.doc_id - self.db1.create_index('test-idx', 'key') - self.sync(self.db1, self.db2) - self.db2.create_index('test-idx', 'key') - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.db1.delete_doc(doc1) - deleted_rev = doc1.rev - self.sync(self.db1, self.db2) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db1, doc_id, deleted_rev, None, False) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, deleted_rev, None, False) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - self.assertEqual([], self.db2.get_from_index('test-idx', 'value')) - self.sync(self.db2, self.db3) - self.assertLastExchangeLog(self.db3, - {'receive': - {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test2', - 'source_gen': 2, - 'last_known_gen': 0}, - 'return': - {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db3, doc_id, deleted_rev, None, False) - - def test_sync_propagates_deletes_2(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') - self.sync(self.db1, self.db2) - doc1_2 = self.db2.get_doc('the-doc') - self.db2.delete_doc(doc1_2) - self.sync(self.db1, self.db2) - self.assertGetDocIncludeDeleted( - self.db1, 'the-doc', doc1_2.rev, None, False) - - def test_sync_detects_identical_replica_uid(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test1', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.assertRaises( - errors.InvalidReplicaUID, self.sync, self.db1, self.db2) - - def test_optional_sync_preserve_json(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - cont1 = '{ "a": 2 }' - cont2 = '{ "b":3}' - self.db1.create_doc_from_json(cont1, doc_id="1") - self.db2.create_doc_from_json(cont2, doc_id="2") - self.sync(self.db1, self.db2) - self.assertEqual(cont1, self.db2.get_doc("1").get_json()) - self.assertEqual(cont2, self.db1.get_doc("2").get_json()) - - def test_sync_propagates_resolution(self): - """ - Test if synchronization propagates resolution. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - db3 = self.create_database('test3', 'both') - self.sync(self.db2, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db2._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.sync(db3, self.db1) - # update on 2 - doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') - self.db2.put_doc(doc2) - self.sync(self.db2, db3) - self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) - # update on 1 - doc1.set_json('{"a": 3}') - self.db1.put_doc(doc1) - # conflicts - self.sync(self.db2, self.db1) - self.sync(db3, self.db1) - self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) - self.assertTrue(db3.get_doc('the-doc').has_conflicts) - # resolve - conflicts = self.db2.get_doc_conflicts('the-doc') - doc4 = self.make_document('the-doc', None, '{"a": 4}') - revs = [doc.rev for doc in conflicts] - self.db2.resolve_doc(doc4, revs) - doc2 = self.db2.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc2.get_json()) - self.assertFalse(doc2.has_conflicts) - self.sync(self.db2, db3) - doc3 = db3.get_doc('the-doc') - if ENC_SCHEME_KEY in doc3.content: - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc3.doc_id) - secret = _crypto.secret - doc3.set_json(decrypt_doc_dict( - doc3.content, - doc3.doc_id, doc3.rev, key, secret)) - self.assertEqual(doc4.get_json(), doc3.get_json()) - self.assertFalse(doc3.has_conflicts) - self.db1.close() - self.db2.close() - db3.close() - - def test_sync_puts_changes(self): - """ - Test if sync puts changes in remote replica. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(tests.simple_doc) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertGetEncryptedDoc( - self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_pulls_changes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(tests.simple_doc) - self.db1.create_index('test-idx', 'key') - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertGetDoc(self.db1, doc.doc_id, doc.rev, - tests.simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [], 'last_known_gen': 0}, - 'return': - {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertEqual([doc], self.db1.get_from_index('test-idx', 'value')) - - def test_sync_supersedes_conflicts(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.create_database('test3', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') - self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') - self.sync(self.db3, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.sync(self.db3, self.db2) - self.assertEqual( - self.db2._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - doc1.set_json('{"a": 2}') - self.db1.put_doc(doc1) - self.sync(self.db3, self.db1) - # original doc1 should have been removed from conflicts - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - - def test_sync_stops_after_get_sync_info(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc) - self.sync(self.db1, self.db2) - - def put_hook(state): - self.fail("Tracehook triggered for %s" % (state,)) - - self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) - - def test_sync_detects_rollback_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) - - def test_sync_detects_rollback_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) - - def test_sync_detects_diverged_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db3, self.db2) - - def test_sync_detects_diverged_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db2) - self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, self.db3) - - def test_sync_detects_rollback_and_divergence_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1_copy, self.db2) - - def test_sync_detects_rollback_and_divergence_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) - - -def make_local_db_and_soledad_target( - test, path='test', - source_replica_uid=uuid4().hex): - test.startTwistedServer() - replica_uid = os.path.basename(path) - db = test.request_state._create_database(replica_uid) - sync_db = test._soledad._sync_db - sync_enc_pool = test._soledad._sync_enc_pool - st = soledad_sync_target( - test, db._dbname, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - return db, st - -target_scenarios = [ - ('leap', { - 'create_db_and_target': make_local_db_and_soledad_target, - 'make_app_with_state': make_soledad_app, - 'do_sync': sync_via_synchronizer_and_soledad}), -] - - -class SQLCipherSyncTargetTests(SoledadDatabaseSyncTargetTests): - - # TODO: implement _set_trace_hook(_shallow) in SoledadHTTPSyncTarget so - # skipped tests can be succesfully executed. - - scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, - target_scenarios)) - - whitebox = False diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py deleted file mode 100644 index 1041367b..00000000 --- a/common/src/leap/soledad/common/tests/test_sync.py +++ /dev/null @@ -1,218 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sync.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import json -import tempfile -import threading -import time - -from urlparse import urljoin -from twisted.internet import defer - -from testscenarios import TestWithScenarios - -from leap.soledad.common import couch -from leap.soledad.client import sync - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer -from leap.soledad.common.tests.u1db_tests import simple_doc -from leap.soledad.common.tests.util import make_token_soledad_app -from leap.soledad.common.tests.util import make_soledad_document_for_test -from leap.soledad.common.tests.util import soledad_sync_target -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.common.tests.test_couch import CouchDBTestCase - - -class InterruptableSyncTestCase( - BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): - - """ - Tests for encrypted sync using Soledad server backed by a couch database. - """ - - @staticmethod - def make_app_with_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def make_app(self): - self.request_state = couch.CouchServerState(self.couch_url) - return self.make_app_with_state(self.request_state) - - def setUp(self): - TestCaseWithServer.setUp(self) - CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - - def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - - def test_interruptable_sync(self): - """ - Test if Soledad can sync many smallfiles. - """ - - self.skipTest("Sync is currently not interruptable.") - - class _SyncInterruptor(threading.Thread): - - """ - A thread meant to interrupt the sync process. - """ - - def __init__(self, soledad, couchdb): - self._soledad = soledad - self._couchdb = couchdb - threading.Thread.__init__(self) - - def run(self): - while db._get_generation() < 2: - # print "WAITING %d" % db._get_generation() - time.sleep(0.1) - self._soledad.stop_sync() - time.sleep(1) - - number_of_docs = 10 - self.startServer() - - # instantiate soledad and create a document - sol = self._soledad_instance( - user='user-uuid', server_url=self.getURL()) - - # ensure remote db exists before syncing - db = couch.CouchDatabase.open_database( - urljoin(self.couch_url, 'user-user-uuid'), - create=True, - ensure_ddocs=True) - - # create interruptor thread - t = _SyncInterruptor(sol, db) - t.start() - - d = sol.get_all_docs() - d.addCallback(lambda results: self.assertEqual([], results[1])) - - def _create_docs(results): - # create many small files - deferreds = [] - for i in range(0, number_of_docs): - deferreds.append(sol.create_doc(json.loads(simple_doc))) - return defer.DeferredList(deferreds) - - # sync with server - d.addCallback(_create_docs) - d.addCallback(lambda _: sol.get_all_docs()) - d.addCallback( - lambda results: self.assertEqual(number_of_docs, len(results[1]))) - d.addCallback(lambda _: sol.sync()) - d.addCallback(lambda _: t.join()) - d.addCallback(lambda _: db.get_all_docs()) - d.addCallback( - lambda results: self.assertNotEqual( - number_of_docs, len(results[1]))) - d.addCallback(lambda _: sol.sync()) - d.addCallback(lambda _: db.get_all_docs()) - d.addCallback( - lambda results: self.assertEqual(number_of_docs, len(results[1]))) - - def _tear_down(results): - db.delete_database() - db.close() - sol.close() - - d.addCallback(_tear_down) - return d - - -class TestSoledadDbSync( - TestWithScenarios, - SoledadWithCouchServerMixin, - tests.TestCaseWithServer): - - """ - Test db.sync remote sync shortcut - """ - - scenarios = [ - ('py-token-http', { - 'make_app_with_state': make_token_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, - 'token': True - }), - ] - - oauth = False - token = False - - def setUp(self): - """ - Need to explicitely invoke inicialization on all bases. - """ - SoledadWithCouchServerMixin.setUp(self) - self.startTwistedServer() - self.db = self.make_database_for_test(self, 'test1') - self.db2 = self.request_state._create_database(replica_uid='test') - - def tearDown(self): - """ - Need to explicitely invoke destruction on all bases. - """ - SoledadWithCouchServerMixin.tearDown(self) - # tests.TestCaseWithServer.tearDown(self) - - def do_sync(self): - """ - Perform sync using SoledadSynchronizer, SoledadSyncTarget - and Token auth. - """ - target = soledad_sync_target( - self, self.db2._dbname, - source_replica_uid=self._soledad._dbpool.replica_uid) - self.addCleanup(target.close) - return sync.SoledadSynchronizer( - self.db, - target).sync(defer_decryption=False) - - @defer.inlineCallbacks - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - - doc1 = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - - local_gen_before_sync = yield self.do_sync() - gen, _, changes = self.db.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) - - # TODO: add u1db.tests.test_sync.TestRemoteSyncIntegration diff --git a/common/src/leap/soledad/common/tests/test_sync_deferred.py b/common/src/leap/soledad/common/tests/test_sync_deferred.py deleted file mode 100644 index c62bd156..00000000 --- a/common/src/leap/soledad/common/tests/test_sync_deferred.py +++ /dev/null @@ -1,196 +0,0 @@ -# test_sync_deferred.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test Leap backend bits: sync with deferred encryption/decryption. -""" -import time -import os -import random -import string -import shutil - -from urlparse import urljoin - -from twisted.internet import defer - -from leap.soledad.common import couch - -from leap.soledad.client import sync -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import SQLCipherDatabase - -from testscenarios import TestWithScenarios - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.util import ADDRESS -from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.tests.util import soledad_sync_target - - -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = True - -WAIT_STEP = 1 -MAX_WAIT = 10 -DBPASS = "pass" - - -class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): - - """ - Another base class for testing the deferred encryption/decryption during - the syncs, using the intermediate database. - """ - defer_sync_encryption = True - - def setUp(self): - SoledadWithCouchServerMixin.setUp(self) - self.startTwistedServer() - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - os.unlink(self.db1_file) - self.db_pass = DBPASS - self.email = ADDRESS - - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - - # open test dbs: db1 will be the local sqlcipher db (which - # instantiates a syncdb). We use the self._soledad instance that was - # already created on some setUp method. - import binascii - tohex = binascii.b2a_hex - key = tohex(self._soledad.secrets.get_local_storage_key()) - sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) - dbpath = self._soledad._local_db_path - - self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False, - defer_encryption=True, sync_db_key=sync_db_key) - self.db1 = SQLCipherDatabase(self.opts) - - self.db2 = self.request_state._create_database('test') - - def tearDown(self): - # XXX should not access "private" attrs - shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) - SoledadWithCouchServerMixin.tearDown(self) - - -class SyncTimeoutError(Exception): - - """ - Dummy exception to notify timeout during sync. - """ - pass - - -class TestSoledadDbSyncDeferredEncDecr( - TestWithScenarios, - BaseSoledadDeferredEncTest, - tests.TestCaseWithServer): - - """ - Test db.sync remote sync shortcut. - Case with deferred encryption and decryption: using the intermediate - syncdb. - """ - - scenarios = [ - ('http', { - 'make_app_with_state': make_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, - }), - ] - - oauth = False - token = True - - def setUp(self): - """ - Need to explicitely invoke inicialization on all bases. - """ - BaseSoledadDeferredEncTest.setUp(self) - self.server = self.server_thread = None - self.syncer = None - - def tearDown(self): - """ - Need to explicitely invoke destruction on all bases. - """ - dbsyncer = getattr(self, 'dbsyncer', None) - if dbsyncer: - dbsyncer.close() - BaseSoledadDeferredEncTest.tearDown(self) - - def do_sync(self): - """ - Perform sync using SoledadSynchronizer, SoledadSyncTarget - and Token auth. - """ - replica_uid = self._soledad._dbpool.replica_uid - sync_db = self._soledad._sync_db - sync_enc_pool = self._soledad._sync_enc_pool - dbsyncer = self._soledad._dbsyncer # Soledad.sync uses the dbsyncer - - target = soledad_sync_target( - self, self.db2._dbname, - source_replica_uid=replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - self.addCleanup(target.close) - return sync.SoledadSynchronizer( - dbsyncer, - target).sync(defer_decryption=True) - - def wait_for_sync(self): - """ - Wait for sync to finish. - """ - wait = 0 - syncer = self.syncer - if syncer is not None: - while syncer.syncing: - time.sleep(WAIT_STEP) - wait += WAIT_STEP - if wait >= MAX_WAIT: - raise SyncTimeoutError - - @defer.inlineCallbacks - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = yield self.do_sync() - - gen, _, changes = self.db1.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) diff --git a/common/src/leap/soledad/common/tests/test_sync_mutex.py b/common/src/leap/soledad/common/tests/test_sync_mutex.py deleted file mode 100644 index 973a8587..00000000 --- a/common/src/leap/soledad/common/tests/test_sync_mutex.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sync_mutex.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Test that synchronization is a critical section and, as such, there might not -be two concurrent synchronization processes at the same time. -""" - - -import time -import uuid -import tempfile -import shutil - -from urlparse import urljoin - -from twisted.internet import defer - -from leap.soledad.client.sync import SoledadSynchronizer - -from leap.soledad.common.couch.state import CouchServerState -from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer -from leap.soledad.common.tests.test_couch import CouchDBTestCase - -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.tests.util import make_token_soledad_app -from leap.soledad.common.tests.util import make_soledad_document_for_test -from leap.soledad.common.tests.util import soledad_sync_target - - -# monkey-patch the soledad synchronizer so it stores start and finish times - -_old_sync = SoledadSynchronizer.sync - - -def _timed_sync(self, defer_decryption=True): - t = time.time() - - sync_id = uuid.uuid4() - - if not getattr(self.source, 'sync_times', False): - self.source.sync_times = {} - - self.source.sync_times[sync_id] = {'start': t} - - def _store_finish_time(passthrough): - t = time.time() - self.source.sync_times[sync_id]['end'] = t - return passthrough - - d = _old_sync(self, defer_decryption=defer_decryption) - d.addBoth(_store_finish_time) - return d - -SoledadSynchronizer.sync = _timed_sync - -# -- end of monkey-patching - - -class TestSyncMutex( - BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): - - @staticmethod - def make_app_with_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def make_app(self): - self.request_state = CouchServerState(self.couch_url) - return self.make_app_with_state(self.request_state) - - def setUp(self): - TestCaseWithServer.setUp(self) - CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.user = ('user-%s' % uuid.uuid4().hex) - - def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - shutil.rmtree(self.tempdir) - - def test_two_concurrent_syncs_do_not_overlap_no_docs(self): - self.startServer() - - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self.couch_url, 'user-' + self.user), - create=True, - ensure_ddocs=True) - - sol = self._soledad_instance( - user=self.user, server_url=self.getURL()) - - d1 = sol.sync() - d2 = sol.sync() - - def _assert_syncs_do_not_overlap(thearg): - # recover sync times - sync_times = [] - for key in sol._dbsyncer.sync_times: - sync_times.append(sol._dbsyncer.sync_times[key]) - sync_times.sort(key=lambda s: s['start']) - - self.assertTrue( - (sync_times[0]['start'] < sync_times[0]['end'] and - sync_times[0]['end'] < sync_times[1]['start'] and - sync_times[1]['start'] < sync_times[1]['end'])) - - db.delete_database() - db.close() - sol.close() - - d = defer.gatherResults([d1, d2]) - d.addBoth(_assert_syncs_do_not_overlap) - return d diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py deleted file mode 100644 index f25e84dd..00000000 --- a/common/src/leap/soledad/common/tests/test_sync_target.py +++ /dev/null @@ -1,956 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sync_target.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Test Leap backend bits: sync target -""" -import cStringIO -import os -import time -import json -import u1db -import random -import string -import shutil -from uuid import uuid4 - -from testscenarios import TestWithScenarios -from twisted.internet import defer - -from leap.soledad.client import http_target as target -from leap.soledad.client import crypto -from leap.soledad.client.sqlcipher import SQLCipherU1DBSync -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import SQLCipherDatabase - -from leap.soledad.common.document import SoledadDocument - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.util import make_sqlcipher_database_for_test -from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.tests.util import make_token_soledad_app -from leap.soledad.common.tests.util import make_soledad_document_for_test -from leap.soledad.common.tests.util import soledad_sync_target -from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.common.tests.util import ADDRESS - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_remote_sync_target`. -# ----------------------------------------------------------------------------- - -class TestSoledadParseReceivedDocResponse(SoledadWithCouchServerMixin): - - """ - Some tests had to be copied to this class so we can instantiate our own - target. - """ - - def setUp(self): - SoledadWithCouchServerMixin.setUp(self) - creds = {'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }} - self.target = target.SoledadHTTPSyncTarget( - self.couch_url, - uuid4().hex, - creds, - self._soledad._crypto, - None) - - def tearDown(self): - self.target.close() - SoledadWithCouchServerMixin.tearDown(self) - - def test_extra_comma(self): - """ - Test adapted to use encrypted content. - """ - doc = SoledadDocument('i', rev='r') - doc.content = {} - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc.doc_id) - secret = _crypto.secret - - enc_json = crypto.encrypt_docstr( - doc.get_json(), doc.doc_id, doc.rev, - key, secret) - - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n{},\r\n]") - - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( - ('[\r\n{},\r\n{"id": "i", "rev": "r", ' + - '"content": %s, "gen": 3, "trans_id": "T-sid"}' + - ',\r\n]') % json.dumps(enc_json)) - - def test_wrong_start(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("{}\r\n]") - - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("\r\n{}\r\n]") - - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("") - - def test_wrong_end(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n{}") - - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n") - - def test_missing_comma(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( - '[\r\n{}\r\n{"id": "i", "rev": "r", ' - '"content": "c", "gen": 3}\r\n]') - - def test_no_entries(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n]") - - def test_error_in_stream(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( - '[\r\n{"new_generation": 0},' - '\r\n{"error": "unavailable"}\r\n') - - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( - '[\r\n{"error": "unavailable"}\r\n') - - with self.assertRaises(u1db.errors.BrokenSyncStream): - self.target._parse_received_doc_response('[\r\n{"error": "?"}\r\n') - -# -# functions for TestRemoteSyncTargets -# - - -def make_local_db_and_soledad_target( - test, path='test', - source_replica_uid=uuid4().hex): - test.startTwistedServer() - replica_uid = os.path.basename(path) - db = test.request_state._create_database(replica_uid) - sync_db = test._soledad._sync_db - sync_enc_pool = test._soledad._sync_enc_pool - st = soledad_sync_target( - test, db._dbname, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - return db, st - - -def make_local_db_and_token_soledad_target( - test, - source_replica_uid=uuid4().hex): - db, st = make_local_db_and_soledad_target( - test, path='test', - source_replica_uid=source_replica_uid) - st.set_token_credentials('user-uuid', 'auth-token') - return db, st - - -class TestSoledadSyncTarget( - TestWithScenarios, - SoledadWithCouchServerMixin, - tests.TestCaseWithServer): - - scenarios = [ - ('token_soledad', - {'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_soledad_document_for_test, - 'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_database_for_test': make_sqlcipher_database_for_test, - 'sync_target': soledad_sync_target}), - ] - - def getSyncTarget(self, path=None, source_replica_uid=uuid4().hex): - if self.port is None: - self.startTwistedServer() - sync_db = self._soledad._sync_db - sync_enc_pool = self._soledad._sync_enc_pool - if path is None: - path = self.db2._dbname - target = self.sync_target( - self, path, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - self.addCleanup(target.close) - return target - - def setUp(self): - TestWithScenarios.setUp(self) - SoledadWithCouchServerMixin.setUp(self) - self.startTwistedServer() - self.db1 = make_sqlcipher_database_for_test(self, 'test1') - self.db2 = self.request_state._create_database('test') - - def tearDown(self): - # db2, _ = self.request_state.ensure_database('test2') - self.delete_db(self.db2._dbname) - self.db1.close() - SoledadWithCouchServerMixin.tearDown(self) - TestWithScenarios.tearDown(self) - - @defer.inlineCallbacks - def test_sync_exchange_send(self): - """ - Test for sync exchanging send of document. - - This test was adapted to decrypt remote content before assert. - """ - db = self.db2 - remote_target = self.getSyncTarget() - other_docs = [] - - def receive_doc(doc, gen, trans_id): - other_docs.append((doc.doc_id, doc.rev, doc.get_json())) - - doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - defer_decryption=False) - self.assertEqual(1, new_gen) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', False) - - @defer.inlineCallbacks - def test_sync_exchange_send_failure_and_retry_scenario(self): - """ - Test for sync exchange failure and retry. - - This test was adapted to decrypt remote content before assert. - """ - - def blackhole_getstderr(inst): - return cStringIO.StringIO() - - db = self.db2 - _put_doc_if_newer = db._put_doc_if_newer - trigger_ids = ['doc-here2'] - - def bomb_put_doc_if_newer(self, doc, save_conflict, - replica_uid=None, replica_gen=None, - replica_trans_id=None, number_of_docs=None, - doc_idx=None, sync_id=None): - if doc.doc_id in trigger_ids: - raise u1db.errors.U1DBError - return _put_doc_if_newer(doc, save_conflict=save_conflict, - replica_uid=replica_uid, - replica_gen=replica_gen, - replica_trans_id=replica_trans_id, - number_of_docs=number_of_docs, - doc_idx=doc_idx, sync_id=sync_id) - from leap.soledad.common.backend import SoledadBackend - self.patch( - SoledadBackend, '_put_doc_if_newer', bomb_put_doc_if_newer) - remote_target = self.getSyncTarget( - source_replica_uid='replica') - other_changes = [] - - def receive_doc(doc, gen, trans_id): - other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - doc2 = self.make_document('doc-here2', 'replica:1', - '{"value": "here2"}') - - with self.assertRaises(u1db.errors.U1DBError): - yield remote_target.sync_exchange( - [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], - 'replica', - last_known_generation=0, - last_known_trans_id=None, - insert_doc_cb=receive_doc, - defer_decryption=False) - - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', - False) - self.assertEqual( - (10, 'T-sid'), db._get_replica_gen_and_trans_id('replica')) - self.assertEqual([], other_changes) - # retry - trigger_ids = [] - new_gen, trans_id = yield remote_target.sync_exchange( - [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - defer_decryption=False) - self.assertGetEncryptedDoc( - db, 'doc-here2', 'replica:1', '{"value": "here2"}', - False) - self.assertEqual( - (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica')) - self.assertEqual(2, new_gen) - self.assertEqual( - ('doc-here', 'replica:1', '{"value": "here"}', 1), - other_changes[0][:-1]) - - @defer.inlineCallbacks - def test_sync_exchange_send_ensure_callback(self): - """ - Test for sync exchange failure and retry. - - This test was adapted to decrypt remote content before assert. - """ - remote_target = self.getSyncTarget() - other_docs = [] - replica_uid_box = [] - - def receive_doc(doc, gen, trans_id): - other_docs.append((doc.doc_id, doc.rev, doc.get_json())) - - def ensure_cb(replica_uid): - replica_uid_box.append(replica_uid) - - doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - ensure_callback=ensure_cb, defer_decryption=False) - self.assertEqual(1, new_gen) - db = self.db2 - self.assertEqual(1, len(replica_uid_box)) - self.assertEqual(db._replica_uid, replica_uid_box[0]) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', False) - - def test_sync_exchange_in_stream_error(self): - self.skipTest("bypass this test because our sync_exchange process " - "does not return u1db error 503 \"unavailable\" for " - "now") - - @defer.inlineCallbacks - def test_get_sync_info(self): - db = self.db2 - db._set_replica_gen_and_trans_id('other-id', 1, 'T-transid') - remote_target = self.getSyncTarget( - source_replica_uid='other-id') - sync_info = yield remote_target.get_sync_info('other-id') - self.assertEqual( - ('test', 0, '', 1, 'T-transid'), - sync_info) - - @defer.inlineCallbacks - def test_record_sync_info(self): - remote_target = self.getSyncTarget( - source_replica_uid='other-id') - yield remote_target.record_sync_info('other-id', 2, 'T-transid') - self.assertEqual((2, 'T-transid'), - self.db2._get_replica_gen_and_trans_id('other-id')) - - @defer.inlineCallbacks - def test_sync_exchange_receive(self): - db = self.db2 - doc = db.create_doc_from_json('{"value": "there"}') - remote_target = self.getSyncTarget() - other_changes = [] - - def receive_doc(doc, gen, trans_id): - other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - new_gen, trans_id = yield remote_target.sync_exchange( - [], 'replica', last_known_generation=0, last_known_trans_id=None, - insert_doc_cb=receive_doc) - self.assertEqual(1, new_gen) - self.assertEqual( - (doc.doc_id, doc.rev, '{"value": "there"}', 1), - other_changes[0][:-1]) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -# ----------------------------------------------------------------------------- - -target_scenarios = [ - ('mem,token_soledad', - {'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_app_with_state': make_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, - 'copy_database_for_test': tests.copy_memory_database_for_test, - 'make_document_for_test': tests.make_document_for_test}) -] - - -class SoledadDatabaseSyncTargetTests( - TestWithScenarios, - SoledadWithCouchServerMixin, - tests.DatabaseBaseTests, - tests.TestCaseWithServer): - """ - Adaptation of u1db.tests.test_sync.DatabaseSyncTargetTests. - """ - - # TODO: implement _set_trace_hook(_shallow) in SoledadHTTPSyncTarget so - # skipped tests can be succesfully executed. - - scenarios = target_scenarios - - whitebox = False - - def setUp(self): - tests.TestCaseWithServer.setUp(self) - self.other_changes = [] - SoledadWithCouchServerMixin.setUp(self) - self.db, self.st = make_local_db_and_soledad_target(self) - - def tearDown(self): - self.db.close() - self.st.close() - tests.TestCaseWithServer.tearDown(self) - SoledadWithCouchServerMixin.tearDown(self) - - def set_trace_hook(self, callback, shallow=False): - setter = (self.st._set_trace_hook if not shallow else - self.st._set_trace_hook_shallow) - try: - setter(callback) - except NotImplementedError: - self.skipTest("%s does not implement _set_trace_hook" - % (self.st.__class__.__name__,)) - - @defer.inlineCallbacks - def test_sync_exchange(self): - """ - Test sync exchange. - - This test was adapted to decrypt remote content before assert. - """ - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, - 'T-sid')] - new_gen, trans_id = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertTransactionLog(['doc-id'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, last_trans_id)) - sync_info = yield self.st.get_sync_info('replica') - self.assertEqual(10, sync_info[3]) - - @defer.inlineCallbacks - def test_sync_exchange_push_many(self): - """ - Test sync exchange. - - This test was adapted to decrypt remote content before assert. - """ - docs_by_gen = [ - (self.make_document( - 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), - (self.make_document( - 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] - new_gen, trans_id = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) - self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - sync_info = yield self.st.get_sync_info('replica') - self.assertEqual(11, sync_info[3]) - - @defer.inlineCallbacks - def test_sync_exchange_returns_many_new_docs(self): - """ - Test sync exchange. - - This test was adapted to avoid JSON serialization comparison as local - and remote representations might differ. It looks directly at the - doc's contents instead. - """ - doc = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db.create_doc_from_json(tests.nested_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - new_gen, _ = yield self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - self.assertEqual(2, new_gen) - self.assertEqual( - [(doc.doc_id, doc.rev, 1), - (doc2.doc_id, doc2.rev, 2)], - [c[:-3] + c[-2:-1] for c in self.other_changes]) - self.assertEqual( - json.loads(tests.simple_doc), - json.loads(self.other_changes[0][2])) - self.assertEqual( - json.loads(tests.nested_doc), - json.loads(self.other_changes[1][2])) - if self.whitebox: - self.assertEqual( - self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': - [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - - def receive_doc(self, doc, gen, trans_id): - self.other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - def test_get_sync_target(self): - self.assertIsNot(None, self.st) - - @defer.inlineCallbacks - def test_get_sync_info(self): - sync_info = yield self.st.get_sync_info('other') - self.assertEqual( - ('test', 0, '', 0, ''), sync_info) - - @defer.inlineCallbacks - def test_create_doc_updates_sync_info(self): - sync_info = yield self.st.get_sync_info('other') - self.assertEqual( - ('test', 0, '', 0, ''), sync_info) - self.db.create_doc_from_json(tests.simple_doc) - sync_info = yield self.st.get_sync_info('other') - self.assertEqual(1, sync_info[1]) - - @defer.inlineCallbacks - def test_record_sync_info(self): - yield self.st.record_sync_info('replica', 10, 'T-transid') - sync_info = yield self.st.get_sync_info('replica') - self.assertEqual( - ('test', 0, '', 10, 'T-transid'), sync_info) - - @defer.inlineCallbacks - def test_sync_exchange_deleted(self): - doc = self.db.create_doc_from_json('{}') - edit_rev = 'replica:1|' + doc.rev - docs_by_gen = [ - (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] - new_gen, trans_id = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, edit_rev, None, False) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - sync_info = yield self.st.get_sync_info('replica') - self.assertEqual(10, sync_info[3]) - - @defer.inlineCallbacks - def test_sync_exchange_refuses_conflicts(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, - 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, tests.simple_doc, 1), - self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - @defer.inlineCallbacks - def test_sync_exchange_ignores_convergence(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - gen, txid = self.db._get_generation_info() - docs_by_gen = [ - (self.make_document(doc.doc_id, doc.rev, tests.simple_doc), - 10, 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=gen, - last_known_trans_id=txid, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual(([], 1), (self.other_changes, new_gen)) - - @defer.inlineCallbacks - def test_sync_exchange_returns_new_docs(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_gen, _ = yield self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, tests.simple_doc, 1), - self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - @defer.inlineCallbacks - def test_sync_exchange_returns_deleted_docs(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.db.delete_doc(doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - new_gen, _ = yield self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) - self.assertEqual(2, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) - - @defer.inlineCallbacks - def test_sync_exchange_getting_newer_docs(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - @defer.inlineCallbacks - def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): - expected = [] - - def before_whatschanged_cb(state): - if state != 'before whats_changed': - return - cont = '{"key": "cuncurrent"}' - conc_rev = self.db.put_doc( - self.make_document(doc.doc_id, 'test:1|z:2', cont)) - expected.append((doc.doc_id, conc_rev, cont, 3)) - - self.set_trace_hook(before_whatschanged_cb) - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertEqual(expected, [c[:-1] for c in self.other_changes]) - self.assertEqual(3, new_gen) - - @defer.inlineCallbacks - def test_sync_exchange_with_concurrent_updates(self): - - def after_whatschanged_cb(state): - if state != 'after whats_changed': - return - self.db.create_doc_from_json('{"new": "doc"}') - - self.set_trace_hook(after_whatschanged_cb) - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - @defer.inlineCallbacks - def test_sync_exchange_converged_handling(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - docs_by_gen = [ - (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), - (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, - 'T-bar')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - @defer.inlineCallbacks - def test_sync_exchange_detect_incomplete_exchange(self): - def before_get_docs_explode(state): - if state != 'before get_docs': - return - raise u1db.errors.U1DBError("fail") - self.set_trace_hook(before_get_docs_explode) - # suppress traceback printing in the wsgiref server - # self.patch(simple_server.ServerHandler, - # 'log_exception', lambda h, exc_info: None) - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertRaises( - (u1db.errors.U1DBError, u1db.errors.BrokenSyncStream), - self.st.sync_exchange, [], 'other-replica', - last_known_generation=0, last_known_trans_id=None, - insert_doc_cb=self.receive_doc) - - @defer.inlineCallbacks - def test_sync_exchange_doc_ids(self): - sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) - if sync_exchange_doc_ids is None: - self.skipTest("sync_exchange_doc_ids not implemented") - db2 = self.create_database('test2') - doc = db2.create_doc_from_json(tests.simple_doc) - new_gen, trans_id = yield sync_exchange_doc_ids( - db2, [(doc.doc_id, 10, 'T-sid')], 0, None, - insert_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, - tests.simple_doc, False) - self.assertTransactionLog([doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) - - @defer.inlineCallbacks - def test__set_trace_hook(self): - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb) - yield self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - yield self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ], - called) - - @defer.inlineCallbacks - def test__set_trace_hook_shallow(self): - if (self.st._set_trace_hook_shallow == self.st._set_trace_hook or - self.st._set_trace_hook_shallow.im_func == - target.SoledadHTTPSyncTarget._set_trace_hook_shallow.im_func): - # shallow same as full - expected = ['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ] - else: - expected = ['sync_exchange', 'record_sync_info'] - - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb, shallow=True) - yield self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - yield self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(expected, called) - - -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = False - -WAIT_STEP = 1 -MAX_WAIT = 10 -DBPASS = "pass" - - -class SyncTimeoutError(Exception): - - """ - Dummy exception to notify timeout during sync. - """ - pass - - -class TestSoledadDbSync( - TestWithScenarios, - SoledadWithCouchServerMixin, - tests.TestCaseWithServer): - - """Test db.sync remote sync shortcut""" - - scenarios = [ - ('py-token-http', { - 'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_app_with_state': make_token_soledad_app, - 'make_database_for_test': make_sqlcipher_database_for_test, - 'token': True - }), - ] - - oauth = False - token = False - - def setUp(self): - """ - Need to explicitely invoke inicialization on all bases. - """ - SoledadWithCouchServerMixin.setUp(self) - self.server = self.server_thread = None - self.startTwistedServer() - self.syncer = None - - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - os.unlink(self.db1_file) - self.db_pass = DBPASS - self.email = ADDRESS - - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - - # open test dbs: db1 will be the local sqlcipher db (which - # instantiates a syncdb). We use the self._soledad instance that was - # already created on some setUp method. - import binascii - tohex = binascii.b2a_hex - key = tohex(self._soledad.secrets.get_local_storage_key()) - sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) - dbpath = self._soledad._local_db_path - - self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False, - defer_encryption=True, sync_db_key=sync_db_key) - self.db1 = SQLCipherDatabase(self.opts) - - self.db2 = self.request_state._create_database(replica_uid='test') - - def tearDown(self): - """ - Need to explicitely invoke destruction on all bases. - """ - dbsyncer = getattr(self, 'dbsyncer', None) - if dbsyncer: - dbsyncer.close() - self.db1.close() - self.db2.close() - self._soledad.close() - - # XXX should not access "private" attrs - shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) - SoledadWithCouchServerMixin.tearDown(self) - - def do_sync(self, target_name): - """ - Perform sync using SoledadSynchronizer, SoledadSyncTarget - and Token auth. - """ - if self.token: - creds = {'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }} - target_url = self.getURL(self.db2._dbname) - - # get a u1db syncer - crypto = self._soledad._crypto - replica_uid = self.db1._replica_uid - dbsyncer = SQLCipherU1DBSync( - self.opts, - crypto, - replica_uid, - None, - defer_encryption=True) - self.dbsyncer = dbsyncer - return dbsyncer.sync(target_url, - creds=creds, - defer_decryption=DEFER_DECRYPTION) - else: - return self._do_sync(self, target_name) - - def _do_sync(self, target_name): - if self.oauth: - path = '~/' + target_name - extra = dict(creds={'oauth': { - 'consumer_key': tests.consumer1.key, - 'consumer_secret': tests.consumer1.secret, - 'token_key': tests.token1.key, - 'token_secret': tests.token1.secret, - }}) - else: - path = target_name - extra = {} - target_url = self.getURL(path) - return self.db.sync(target_url, **extra) - - def wait_for_sync(self): - """ - Wait for sync to finish. - """ - wait = 0 - syncer = self.syncer - if syncer is not None: - while syncer.syncing: - time.sleep(WAIT_STEP) - wait += WAIT_STEP - if wait >= MAX_WAIT: - raise SyncTimeoutError - - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - d = self.do_sync('test') - - def _assert_successful_sync(results): - import time - # need to give time to the encryption to proceed - # TODO should implement a defer list to subscribe to the - # all-decrypted event - time.sleep(2) - local_gen_before_sync = results - self.wait_for_sync() - - gen, _, changes = self.db1.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) - - d.addCallback(_assert_successful_sync) - return d diff --git a/common/src/leap/soledad/common/tests/u1db_tests/README b/common/src/leap/soledad/common/tests/u1db_tests/README deleted file mode 100644 index 0525cfdb..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/README +++ /dev/null @@ -1,25 +0,0 @@ -General info ------------- - -Test files in this directory are derived from u1db-0.1.4 tests. The main -difference is that: - - (1) they include the test infrastructure packed with soledad; and - (2) they do not include c_backend_wrapper testing. - -Dependencies ------------- - -u1db tests depend on the following python packages: - - unittest2 - mercurial - hgtools - testtools - discover - oauth - testscenarios - dirspec - paste - routes - cython diff --git a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py deleted file mode 100644 index 01da9381..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py +++ /dev/null @@ -1,461 +0,0 @@ -# Copyright 2011-2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. -""" -Test infrastructure for U1DB -""" - -import copy -import shutil -import socket -import tempfile -import threading -import json - -from wsgiref import simple_server - -from oauth import oauth -from pysqlcipher import dbapi2 -from StringIO import StringIO - -import testscenarios -from twisted.trial import unittest -from twisted.web.server import Site -from twisted.web.wsgi import WSGIResource -from twisted.internet import reactor - -from u1db import errors -from u1db import Document -from u1db.backends import inmemory -from u1db.backends import sqlite_backend -from u1db.remote import server_state -from u1db.remote import http_app -from u1db.remote import http_target - - -class TestCase(unittest.TestCase): - - def createTempDir(self, prefix='u1db-tmp-'): - """Create a temporary directory to do some work in. - - This directory will be scheduled for cleanup when the test ends. - """ - tempdir = tempfile.mkdtemp(prefix=prefix) - self.addCleanup(shutil.rmtree, tempdir) - return tempdir - - def make_document(self, doc_id, doc_rev, content, has_conflicts=False): - return self.make_document_for_test( - self, doc_id, doc_rev, content, has_conflicts) - - def make_document_for_test(self, test, doc_id, doc_rev, content, - has_conflicts): - return make_document_for_test( - test, doc_id, doc_rev, content, has_conflicts) - - def assertGetDoc(self, db, doc_id, doc_rev, content, has_conflicts): - """Assert that the document in the database looks correct.""" - exp_doc = self.make_document(doc_id, doc_rev, content, - has_conflicts=has_conflicts) - self.assertEqual(exp_doc, db.get_doc(doc_id)) - - def assertGetDocIncludeDeleted(self, db, doc_id, doc_rev, content, - has_conflicts): - """Assert that the document in the database looks correct.""" - exp_doc = self.make_document(doc_id, doc_rev, content, - has_conflicts=has_conflicts) - self.assertEqual(exp_doc, db.get_doc(doc_id, include_deleted=True)) - - def assertGetDocConflicts(self, db, doc_id, conflicts): - """Assert what conflicts are stored for a given doc_id. - - :param conflicts: A list of (doc_rev, content) pairs. - The first item must match the first item returned from the - database, however the rest can be returned in any order. - """ - if conflicts: - conflicts = [(rev, - (json.loads(cont) if isinstance(cont, basestring) - else cont)) for (rev, cont) in conflicts] - conflicts = conflicts[:1] + sorted(conflicts[1:]) - actual = db.get_doc_conflicts(doc_id) - if actual: - actual = [ - (doc.rev, (json.loads(doc.get_json()) - if doc.get_json() is not None else None)) - for doc in actual] - actual = actual[:1] + sorted(actual[1:]) - self.assertEqual(conflicts, actual) - - -def multiply_scenarios(a_scenarios, b_scenarios): - """Create the cross-product of scenarios.""" - - all_scenarios = [] - for a_name, a_attrs in a_scenarios: - for b_name, b_attrs in b_scenarios: - name = '%s,%s' % (a_name, b_name) - attrs = dict(a_attrs) - attrs.update(b_attrs) - all_scenarios.append((name, attrs)) - return all_scenarios - - -simple_doc = '{"key": "value"}' -nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}' - - -def make_memory_database_for_test(test, replica_uid): - return inmemory.InMemoryDatabase(replica_uid) - - -def copy_memory_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - new_db = inmemory.InMemoryDatabase(db._replica_uid) - new_db._transaction_log = db._transaction_log[:] - new_db._docs = copy.deepcopy(db._docs) - new_db._conflicts = copy.deepcopy(db._conflicts) - new_db._indexes = copy.deepcopy(db._indexes) - new_db._factory = db._factory - return new_db - - -def make_sqlite_partial_expanded_for_test(test, replica_uid): - db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') - db._set_replica_uid(replica_uid) - return db - - -def copy_sqlite_partial_expanded_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - new_db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') - tmpfile = StringIO() - for line in db._db_handle.iterdump(): - if 'sqlite_sequence' not in line: # work around bug in iterdump - tmpfile.write('%s\n' % line) - tmpfile.seek(0) - new_db._db_handle = dbapi2.connect(':memory:') - new_db._db_handle.cursor().executescript(tmpfile.read()) - new_db._db_handle.commit() - new_db._set_replica_uid(db._replica_uid) - new_db._factory = db._factory - return new_db - - -def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): - return Document(doc_id, rev, content, has_conflicts=has_conflicts) - - -LOCAL_DATABASES_SCENARIOS = [ - ('mem', {'make_database_for_test': make_memory_database_for_test, - 'copy_database_for_test': copy_memory_database_for_test, - 'make_document_for_test': make_document_for_test}), - ('sql', {'make_database_for_test': - make_sqlite_partial_expanded_for_test, - 'copy_database_for_test': - copy_sqlite_partial_expanded_for_test, - 'make_document_for_test': make_document_for_test}), -] - - -class DatabaseBaseTests(TestCase): - - # set to True assertTransactionLog - # is happy with all trans ids = '' - accept_fixed_trans_id = False - - scenarios = LOCAL_DATABASES_SCENARIOS - - def make_database_for_test(self, replica_uid): - return make_memory_database_for_test(self, replica_uid) - - def create_database(self, *args): - return self.make_database_for_test(self, *args) - - def copy_database(self, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - return self.copy_database_for_test(self, db) - - def setUp(self): - super(DatabaseBaseTests, self).setUp() - self.db = self.create_database('test') - - def tearDown(self): - if hasattr(self, 'db') and self.db is not None: - self.db.close() - super(DatabaseBaseTests, self).tearDown() - - def assertTransactionLog(self, doc_ids, db): - """Assert that the given docs are in the transaction log.""" - log = db._get_transaction_log() - just_ids = [] - seen_transactions = set() - for doc_id, transaction_id in log: - just_ids.append(doc_id) - self.assertIsNot(None, transaction_id, - "Transaction id should not be None") - if transaction_id == '' and self.accept_fixed_trans_id: - continue - self.assertNotEqual('', transaction_id, - "Transaction id should be a unique string") - self.assertTrue(transaction_id.startswith('T-')) - self.assertNotIn(transaction_id, seen_transactions) - seen_transactions.add(transaction_id) - self.assertEqual(doc_ids, just_ids) - - def getLastTransId(self, db): - """Return the transaction id for the last database update.""" - return self.db._get_transaction_log()[-1][-1] - - -class ServerStateForTests(server_state.ServerState): - - """Used in the test suite, so we don't have to touch disk, etc.""" - - def __init__(self): - super(ServerStateForTests, self).__init__() - self._dbs = {} - - def open_database(self, path): - try: - return self._dbs[path] - except KeyError: - raise errors.DatabaseDoesNotExist - - def check_database(self, path): - # cares only about the possible exception - self.open_database(path) - - def ensure_database(self, path): - try: - db = self.open_database(path) - except errors.DatabaseDoesNotExist: - db = self._create_database(path) - return db, db._replica_uid - - def _copy_database(self, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - new_db = copy_memory_database_for_test(None, db) - path = db._replica_uid - while path in self._dbs: - path += 'copy' - self._dbs[path] = new_db - return new_db - - def _create_database(self, path): - db = inmemory.InMemoryDatabase(path) - self._dbs[path] = db - return db - - def delete_database(self, path): - del self._dbs[path] - - -class ResponderForTests(object): - - """Responder for tests.""" - _started = False - sent_response = False - status = None - - def start_response(self, status='success', **kwargs): - self._started = True - self.status = status - self.kwargs = kwargs - - def send_response(self, status='success', **kwargs): - self.start_response(status, **kwargs) - self.finish_response() - - def finish_response(self): - self.sent_response = True - - -class TestCaseWithServer(TestCase): - - @staticmethod - def server_def(): - # hook point - # should return (ServerClass, "shutdown method name", "url_scheme") - class _RequestHandler(simple_server.WSGIRequestHandler): - - def log_request(*args): - pass # suppress - - def make_server(host_port, application): - assert application, "forgot to override make_app(_with_state)?" - srv = simple_server.WSGIServer(host_port, _RequestHandler) - # patch the value in if it's None - if getattr(application, 'base_url', 1) is None: - application.base_url = "http://%s:%s" % srv.server_address - srv.set_app(application) - return srv - - return make_server, "shutdown", "http" - - @staticmethod - def make_app_with_state(state): - # hook point - return None - - def make_app(self): - # potential hook point - self.request_state = ServerStateForTests() - return self.make_app_with_state(self.request_state) - - def setUp(self): - super(TestCaseWithServer, self).setUp() - self.server = self.server_thread = self.port = None - - def tearDown(self): - if self.server is not None: - self.server.shutdown() - self.server_thread.join() - self.server.server_close() - if self.port: - self.port.stopListening() - super(TestCaseWithServer, self).tearDown() - - @property - def url_scheme(self): - return 'http' - - def startTwistedServer(self): - application = self.make_app() - resource = WSGIResource(reactor, reactor.getThreadPool(), application) - site = Site(resource) - self.port = reactor.listenTCP(0, site, interface='127.0.0.1') - host = self.port.getHost() - self.server_address = (host.host, host.port) - self.addCleanup(self.port.stopListening) - - def startServer(self): - server_def = self.server_def() - server_class, shutdown_meth, _ = server_def - application = self.make_app() - self.server = server_class(('127.0.0.1', 0), application) - self.server_thread = threading.Thread(target=self.server.serve_forever, - kwargs=dict(poll_interval=0.01)) - self.server_thread.start() - self.addCleanup(self.server_thread.join) - self.addCleanup(getattr(self.server, shutdown_meth)) - self.server_address = self.server.server_address - - def getURL(self, path=None): - host, port = self.server_address - if path is None: - path = '' - return '%s://%s:%s/%s' % (self.url_scheme, host, port, path) - - -def socket_pair(): - """Return a pair of TCP sockets connected to each other. - - Unlike socket.socketpair, this should work on Windows. - """ - sock_pair = getattr(socket, 'socket_pair', None) - if sock_pair: - return sock_pair(socket.AF_INET, socket.SOCK_STREAM) - listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listen_sock.bind(('127.0.0.1', 0)) - listen_sock.listen(1) - client_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - client_sock.connect(listen_sock.getsockname()) - server_sock, addr = listen_sock.accept() - listen_sock.close() - return server_sock, client_sock - - -# OAuth related testing - -consumer1 = oauth.OAuthConsumer('K1', 'S1') -token1 = oauth.OAuthToken('kkkk1', 'XYZ') -consumer2 = oauth.OAuthConsumer('K2', 'S2') -token2 = oauth.OAuthToken('kkkk2', 'ZYX') -token3 = oauth.OAuthToken('kkkk3', 'ZYX') - - -class TestingOAuthDataStore(oauth.OAuthDataStore): - - """In memory predefined OAuthDataStore for testing.""" - - consumers = { - consumer1.key: consumer1, - consumer2.key: consumer2, - } - - tokens = { - token1.key: token1, - token2.key: token2 - } - - def lookup_consumer(self, key): - return self.consumers.get(key) - - def lookup_token(self, token_type, token_token): - return self.tokens.get(token_token) - - def lookup_nonce(self, oauth_consumer, oauth_token, nonce): - return None - -testingOAuthStore = TestingOAuthDataStore() - -sign_meth_HMAC_SHA1 = oauth.OAuthSignatureMethod_HMAC_SHA1() -sign_meth_PLAINTEXT = oauth.OAuthSignatureMethod_PLAINTEXT() - - -def load_with_scenarios(loader, standard_tests, pattern): - """Load the tests in a given module. - - This just applies testscenarios.generate_scenarios to all the tests that - are present. We do it at load time rather than at run time, because it - plays nicer with various tools. - """ - suite = loader.suiteClass() - suite.addTests(testscenarios.generate_scenarios(standard_tests)) - return suite - - -# from u1db.tests.test_remote_sync_target - -def make_http_app(state): - return http_app.HTTPApp(state) - - -def http_sync_target(test, path): - return http_target.HTTPSyncTarget(test.getURL(path)) - - -def make_oauth_http_app(state): - app = http_app.HTTPApp(state) - application = oauth_middleware.OAuthMiddleware(app, None, prefix='/~/') - application.get_oauth_data_store = lambda: tests.testingOAuthStore - return application diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py deleted file mode 100644 index 410d838f..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py +++ /dev/null @@ -1,1914 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""The backend class for U1DB. This deals with hiding storage details.""" - -import json - -from u1db import DocumentBase -from u1db import errors -from u1db import vectorclock - -from leap.soledad.common.tests import u1db_tests as tests - -from leap.soledad.common.tests.u1db_tests import make_http_app -from leap.soledad.common.tests.u1db_tests import make_oauth_http_app - -from u1db.remote import http_database - -from unittest import skip - -simple_doc = tests.simple_doc -nested_doc = tests.nested_doc - - -def make_http_database_for_test(test, replica_uid, path='test', *args): - test.startServer() - test.request_state._create_database(replica_uid) - return http_database.HTTPDatabase(test.getURL(path)) - - -def copy_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - return test.request_state._copy_database(db) - - -def make_oauth_http_database_for_test(test, replica_uid): - http_db = make_http_database_for_test(test, replica_uid, '~/test') - http_db.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - return http_db - - -def copy_oauth_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - return http_db - - -class TestAlternativeDocument(DocumentBase): - - """A (not very) alternative implementation of Document.""" - - -@skip("Skiping tests imported from U1DB.") -class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS + [ - ('http', {'make_database_for_test': make_http_database_for_test, - 'copy_database_for_test': copy_http_database_for_test, - 'make_document_for_test': tests.make_document_for_test, - 'make_app_with_state': make_http_app}), - ('oauth_http', {'make_database_for_test': - make_oauth_http_database_for_test, - 'copy_database_for_test': - copy_oauth_http_database_for_test, - 'make_document_for_test': tests.make_document_for_test, - 'make_app_with_state': make_oauth_http_app}) - ] - - def test_close(self): - self.db.close() - - def test_create_doc_allocating_doc_id(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertNotEqual(None, doc.doc_id) - self.assertNotEqual(None, doc.rev) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - - def test_create_doc_different_ids_same_db(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertNotEqual(doc1.doc_id, doc2.doc_id) - - def test_create_doc_with_id(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my-id') - self.assertEqual('my-id', doc.doc_id) - self.assertNotEqual(None, doc.rev) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - - def test_create_doc_existing_id(self): - doc = self.db.create_doc_from_json(simple_doc) - new_content = '{"something": "else"}' - self.assertRaises( - errors.RevisionConflict, self.db.create_doc_from_json, - new_content, doc.doc_id) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - - def test_put_doc_creating_initial(self): - doc = self.make_document('my_doc_id', None, simple_doc) - new_rev = self.db.put_doc(doc) - self.assertIsNot(None, new_rev) - self.assertGetDoc(self.db, 'my_doc_id', new_rev, simple_doc, False) - - def test_put_doc_space_in_id(self): - doc = self.make_document('my doc id', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_put_doc_update(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - orig_rev = doc.rev - doc.set_json('{"updated": "stuff"}') - new_rev = self.db.put_doc(doc) - self.assertNotEqual(new_rev, orig_rev) - self.assertGetDoc(self.db, 'my_doc_id', new_rev, - '{"updated": "stuff"}', False) - self.assertEqual(doc.rev, new_rev) - - def test_put_non_ascii_key(self): - content = json.dumps({u'key\xe5': u'val'}) - doc = self.db.create_doc_from_json(content, doc_id='my_doc') - self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) - - def test_put_non_ascii_value(self): - content = json.dumps({'key': u'\xe5'}) - doc = self.db.create_doc_from_json(content, doc_id='my_doc') - self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) - - def test_put_doc_refuses_no_id(self): - doc = self.make_document(None, None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - doc = self.make_document("", None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_put_doc_refuses_slashes(self): - doc = self.make_document('a/b', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - doc = self.make_document(r'\b', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_put_doc_url_quoting_is_fine(self): - doc_id = "%2F%2Ffoo%2Fbar" - doc = self.make_document(doc_id, None, simple_doc) - new_rev = self.db.put_doc(doc) - self.assertGetDoc(self.db, doc_id, new_rev, simple_doc, False) - - def test_put_doc_refuses_non_existing_old_rev(self): - doc = self.make_document('doc-id', 'test:4', simple_doc) - self.assertRaises(errors.RevisionConflict, self.db.put_doc, doc) - - def test_put_doc_refuses_non_ascii_doc_id(self): - doc = self.make_document('d\xc3\xa5c-id', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_put_fails_with_bad_old_rev(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - old_rev = doc.rev - bad_doc = self.make_document(doc.doc_id, 'other:1', - '{"something": "else"}') - self.assertRaises(errors.RevisionConflict, self.db.put_doc, bad_doc) - self.assertGetDoc(self.db, 'my_doc_id', old_rev, simple_doc, False) - - def test_create_succeeds_after_delete(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.db.delete_doc(doc) - deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) - deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) - new_doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.assertGetDoc(self.db, 'my_doc_id', new_doc.rev, simple_doc, False) - new_vc = vectorclock.VectorClockRev(new_doc.rev) - self.assertTrue( - new_vc.is_newer(deleted_vc), - "%s does not supersede %s" % (new_doc.rev, deleted_doc.rev)) - - def test_put_succeeds_after_delete(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.db.delete_doc(doc) - deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) - deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) - doc2 = self.make_document('my_doc_id', None, simple_doc) - self.db.put_doc(doc2) - self.assertGetDoc(self.db, 'my_doc_id', doc2.rev, simple_doc, False) - new_vc = vectorclock.VectorClockRev(doc2.rev) - self.assertTrue( - new_vc.is_newer(deleted_vc), - "%s does not supersede %s" % (doc2.rev, deleted_doc.rev)) - - def test_get_doc_after_put(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.assertGetDoc(self.db, 'my_doc_id', doc.rev, simple_doc, False) - - def test_get_doc_nonexisting(self): - self.assertIs(None, self.db.get_doc('non-existing')) - - def test_get_doc_deleted(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.db.delete_doc(doc) - self.assertIs(None, self.db.get_doc('my_doc_id')) - - def test_get_doc_include_deleted(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.db.delete_doc(doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, doc.rev, None, False) - - def test_get_docs(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertEqual([doc1, doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) - - def test_get_docs_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.db.delete_doc(doc1) - self.assertEqual([doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) - - def test_get_docs_include_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.db.delete_doc(doc1) - self.assertEqual( - [doc1, doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id], - include_deleted=True))) - - def test_get_docs_request_ordered(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertEqual([doc1, doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) - self.assertEqual([doc2, doc1], - list(self.db.get_docs([doc2.doc_id, doc1.doc_id]))) - - def test_get_docs_empty_list(self): - self.assertEqual([], list(self.db.get_docs([]))) - - def test_handles_nested_content(self): - doc = self.db.create_doc_from_json(nested_doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) - - def test_handles_doc_with_null(self): - doc = self.db.create_doc_from_json('{"key": null}') - self.assertGetDoc(self.db, doc.doc_id, doc.rev, '{"key": null}', False) - - def test_delete_doc(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - orig_rev = doc.rev - self.db.delete_doc(doc) - self.assertNotEqual(orig_rev, doc.rev) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, doc.rev, None, False) - self.assertIs(None, self.db.get_doc(doc.doc_id)) - - def test_delete_doc_non_existent(self): - doc = self.make_document('non-existing', 'other:1', simple_doc) - self.assertRaises(errors.DocumentDoesNotExist, self.db.delete_doc, doc) - - def test_delete_doc_already_deleted(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc) - self.assertRaises(errors.DocumentAlreadyDeleted, - self.db.delete_doc, doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, doc.rev, None, False) - - def test_delete_doc_bad_rev(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) - doc2 = self.make_document(doc1.doc_id, 'other:1', simple_doc) - self.assertRaises(errors.RevisionConflict, self.db.delete_doc, doc2) - self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) - - def test_delete_doc_sets_content_to_None(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc) - self.assertIs(None, doc.get_json()) - - def test_delete_doc_rev_supersedes(self): - doc = self.db.create_doc_from_json(simple_doc) - doc.set_json(nested_doc) - self.db.put_doc(doc) - doc.set_json('{"fishy": "content"}') - self.db.put_doc(doc) - old_rev = doc.rev - self.db.delete_doc(doc) - cur_vc = vectorclock.VectorClockRev(old_rev) - deleted_vc = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(deleted_vc.is_newer(cur_vc), - "%s does not supersede %s" % (doc.rev, old_rev)) - - def test_delete_then_put(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, doc.rev, None, False) - doc.set_json(nested_doc) - self.db.put_doc(doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) - - -@skip("Skiping tests imported from U1DB.") -class DocumentSizeTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def test_put_doc_refuses_oversized_documents(self): - self.db.set_document_size_limit(1) - doc = self.make_document('doc-id', None, simple_doc) - self.assertRaises(errors.DocumentTooBig, self.db.put_doc, doc) - - def test_create_doc_refuses_oversized_documents(self): - self.db.set_document_size_limit(1) - self.assertRaises( - errors.DocumentTooBig, self.db.create_doc_from_json, simple_doc, - doc_id='my_doc_id') - - def test_set_document_size_limit_zero(self): - self.db.set_document_size_limit(0) - self.assertEqual(0, self.db.document_size_limit) - - def test_set_document_size_limit(self): - self.db.set_document_size_limit(1000000) - self.assertEqual(1000000, self.db.document_size_limit) - - -@skip("Skiping tests imported from U1DB.") -class LocalDatabaseTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def setUp(self): - tests.DatabaseBaseTests.setUp(self) - - def test_create_doc_different_ids_diff_db(self): - doc1 = self.db.create_doc_from_json(simple_doc) - db2 = self.create_database('other-uid') - doc2 = db2.create_doc_from_json(simple_doc) - self.assertNotEqual(doc1.doc_id, doc2.doc_id) - db2.close() - - def test_put_doc_refuses_slashes_picky(self): - doc = self.make_document('/a', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_get_all_docs_empty(self): - self.assertEqual([], list(self.db.get_all_docs()[1])) - - def test_get_all_docs(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertEqual( - sorted([doc1, doc2]), sorted(list(self.db.get_all_docs()[1]))) - - def test_get_all_docs_exclude_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.db.delete_doc(doc2) - self.assertEqual([doc1], list(self.db.get_all_docs()[1])) - - def test_get_all_docs_include_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.db.delete_doc(doc2) - self.assertEqual( - sorted([doc1, doc2]), - sorted(list(self.db.get_all_docs(include_deleted=True)[1]))) - - def test_get_all_docs_generation(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_doc_from_json(nested_doc) - self.assertEqual(2, self.db.get_all_docs()[0]) - - def test_simple_put_doc_if_newer(self): - doc = self.make_document('my-doc-id', 'test:1', simple_doc) - state_at_gen = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(('inserted', 1), state_at_gen) - self.assertGetDoc(self.db, 'my-doc-id', 'test:1', simple_doc, False) - - def test_simple_put_doc_if_newer_deleted(self): - self.db.create_doc_from_json('{}', doc_id='my-doc-id') - doc = self.make_document('my-doc-id', 'test:2', None) - state_at_gen = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(('inserted', 2), state_at_gen) - self.assertGetDocIncludeDeleted( - self.db, 'my-doc-id', 'test:2', None, False) - - def test_put_doc_if_newer_already_superseded(self): - orig_doc = '{"new": "doc"}' - doc1 = self.db.create_doc_from_json(orig_doc) - doc1_rev1 = doc1.rev - doc1.set_json(simple_doc) - self.db.put_doc(doc1) - doc1_rev2 = doc1.rev - # Nothing is inserted, because the document is already superseded - doc = self.make_document(doc1.doc_id, doc1_rev1, orig_doc) - state, _ = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual('superseded', state) - self.assertGetDoc(self.db, doc1.doc_id, doc1_rev2, simple_doc, False) - - def test_put_doc_if_newer_autoresolve(self): - doc1 = self.db.create_doc_from_json(simple_doc) - rev = doc1.rev - doc = self.make_document(doc1.doc_id, "whatever:1", doc1.get_json()) - state, _ = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual('superseded', state) - doc2 = self.db.get_doc(doc1.doc_id) - v2 = vectorclock.VectorClockRev(doc2.rev) - self.assertTrue(v2.is_newer(vectorclock.VectorClockRev("whatever:1"))) - self.assertTrue(v2.is_newer(vectorclock.VectorClockRev(rev))) - # strictly newer locally - self.assertTrue(rev not in doc2.rev) - - def test_put_doc_if_newer_already_converged(self): - orig_doc = '{"new": "doc"}' - doc1 = self.db.create_doc_from_json(orig_doc) - state_at_gen = self.db._put_doc_if_newer( - doc1, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(('converged', 1), state_at_gen) - - def test_put_doc_if_newer_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - # Nothing is inserted, the document id is returned as would-conflict - alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - state, _ = self.db._put_doc_if_newer( - alt_doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual('conflicted', state) - # The database wasn't altered - self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) - - def test_put_doc_if_newer_newer_generation(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - doc = self.make_document('doc_id', 'other:2', simple_doc) - state, _ = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='other', replica_gen=2, - replica_trans_id='T-irrelevant') - self.assertEqual('inserted', state) - - def test_put_doc_if_newer_same_generation_same_txid(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - doc = self.db.create_doc_from_json(simple_doc) - self.make_document(doc.doc_id, 'other:1', simple_doc) - state, _ = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='other', replica_gen=1, - replica_trans_id='T-sid') - self.assertEqual('converged', state) - - def test_put_doc_if_newer_wrong_transaction_id(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - doc = self.make_document('doc_id', 'other:1', simple_doc) - self.assertRaises( - errors.InvalidTransactionId, - self.db._put_doc_if_newer, doc, save_conflict=False, - replica_uid='other', replica_gen=1, replica_trans_id='T-sad') - - def test_put_doc_if_newer_old_generation_older_doc(self): - orig_doc = '{"new": "doc"}' - doc = self.db.create_doc_from_json(orig_doc) - doc_rev1 = doc.rev - doc.set_json(simple_doc) - self.db.put_doc(doc) - self.db._set_replica_gen_and_trans_id('other', 3, 'T-sid') - older_doc = self.make_document(doc.doc_id, doc_rev1, simple_doc) - state, _ = self.db._put_doc_if_newer( - older_doc, save_conflict=False, replica_uid='other', replica_gen=8, - replica_trans_id='T-irrelevant') - self.assertEqual('superseded', state) - - def test_put_doc_if_newer_old_generation_newer_doc(self): - self.db._set_replica_gen_and_trans_id('other', 5, 'T-sid') - doc = self.make_document('doc_id', 'other:1', simple_doc) - self.assertRaises( - errors.InvalidGeneration, - self.db._put_doc_if_newer, doc, save_conflict=False, - replica_uid='other', replica_gen=1, replica_trans_id='T-sad') - - def test_put_doc_if_newer_replica_uid(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', - nested_doc) - self.assertEqual('inserted', - self.db._put_doc_if_newer( - doc2, - save_conflict=False, - replica_uid='other', - replica_gen=2, - replica_trans_id='T-id2')[0]) - self.assertEqual((2, 'T-id2'), self.db._get_replica_gen_and_trans_id( - 'other')) - # Compare to the old rev, should be superseded - doc2 = self.make_document(doc1.doc_id, doc1.rev, nested_doc) - self.assertEqual('superseded', - self.db._put_doc_if_newer( - doc2, - save_conflict=False, - replica_uid='other', - replica_gen=3, - replica_trans_id='T-id3')[0]) - self.assertEqual( - (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) - # A conflict that isn't saved still records the sync gen, because we - # don't need to see it again - doc2 = self.make_document(doc1.doc_id, doc1.rev + '|fourth:1', - '{}') - self.assertEqual('conflicted', - self.db._put_doc_if_newer( - doc2, - save_conflict=False, - replica_uid='other', - replica_gen=4, - replica_trans_id='T-id4')[0]) - self.assertEqual( - (4, 'T-id4'), self.db._get_replica_gen_and_trans_id('other')) - - def test__get_replica_gen_and_trans_id(self): - self.assertEqual( - (0, ''), self.db._get_replica_gen_and_trans_id('other-db')) - self.db._set_replica_gen_and_trans_id('other-db', 2, 'T-transaction') - self.assertEqual( - (2, 'T-transaction'), - self.db._get_replica_gen_and_trans_id('other-db')) - - def test_put_updates_transaction_log(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - doc.set_json('{"something": "else"}') - self.db.put_doc(doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), - self.db.whats_changed()) - - def test_delete_updates_transaction_log(self): - doc = self.db.create_doc_from_json(simple_doc) - db_gen, _, _ = self.db.whats_changed() - self.db.delete_doc(doc) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), - self.db.whats_changed(db_gen)) - - def test_whats_changed_initial_database(self): - self.assertEqual((0, '', []), self.db.whats_changed()) - - def test_whats_changed_returns_one_id_for_multiple_changes(self): - doc = self.db.create_doc_from_json(simple_doc) - doc.set_json('{"new": "contents"}') - self.db.put_doc(doc) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), - self.db.whats_changed()) - self.assertEqual((2, last_trans_id, []), self.db.whats_changed(2)) - - def test_whats_changed_returns_last_edits_ascending(self): - doc = self.db.create_doc_from_json(simple_doc) - doc1 = self.db.create_doc_from_json(simple_doc) - doc.set_json('{"new": "contents"}') - self.db.delete_doc(doc1) - delete_trans_id = self.getLastTransId(self.db) - self.db.put_doc(doc) - put_trans_id = self.getLastTransId(self.db) - self.assertEqual((4, put_trans_id, - [(doc1.doc_id, 3, delete_trans_id), - (doc.doc_id, 4, put_trans_id)]), - self.db.whats_changed()) - - def test_whats_changed_doesnt_include_old_gen(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(simple_doc) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual((3, last_trans_id, [(doc2.doc_id, 3, last_trans_id)]), - self.db.whats_changed(2)) - - -@skip("Skiping tests imported from U1DB.") -class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def test_validate_gen_and_trans_id(self): - self.db.create_doc_from_json(simple_doc) - gen, trans_id = self.db._get_generation_info() - self.db.validate_gen_and_trans_id(gen, trans_id) - - def test_validate_gen_and_trans_id_invalid_txid(self): - self.db.create_doc_from_json(simple_doc) - gen, _ = self.db._get_generation_info() - self.assertRaises( - errors.InvalidTransactionId, - self.db.validate_gen_and_trans_id, gen, 'wrong') - - def test_validate_gen_and_trans_id_invalid_gen(self): - self.db.create_doc_from_json(simple_doc) - gen, trans_id = self.db._get_generation_info() - self.assertRaises( - errors.InvalidGeneration, - self.db.validate_gen_and_trans_id, gen + 1, trans_id) - - -@skip("Skiping tests imported from U1DB.") -class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def test_validate_source_gen_and_trans_id_same(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - self.db._validate_source('other', 1, 'T-sid') - - def test_validate_source_gen_newer(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - self.db._validate_source('other', 2, 'T-whatevs') - - def test_validate_source_wrong_txid(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - self.assertRaises( - errors.InvalidTransactionId, - self.db._validate_source, 'other', 1, 'T-sad') - - -@skip("Skiping tests imported from U1DB.") -class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): - # test supporting/functionality around storing conflicts - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def test_get_docs_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual([doc2], list(self.db.get_docs([doc1.doc_id]))) - - def test_get_docs_conflicts_ignored(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - no_conflict_doc = self.make_document(doc1.doc_id, 'alternate:1', - nested_doc) - self.assertEqual([no_conflict_doc, doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id], - check_for_conflicts=False))) - - def test_get_doc_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual([alt_doc, doc], - self.db.get_doc_conflicts(doc.doc_id)) - - def test_get_all_docs_sees_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - _, docs = self.db.get_all_docs() - self.assertTrue(list(docs)[0].has_conflicts) - - def test_get_doc_conflicts_unconflicted(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertEqual([], self.db.get_doc_conflicts(doc.doc_id)) - - def test_get_doc_conflicts_no_such_id(self): - self.assertEqual([], self.db.get_doc_conflicts('doc-id')) - - def test_resolve_doc(self): - doc = self.db.create_doc_from_json(simple_doc) - alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc.doc_id, - [('alternate:1', nested_doc), - (doc.rev, simple_doc)]) - orig_rev = doc.rev - self.db.resolve_doc(doc, [alt_doc.rev, doc.rev]) - self.assertNotEqual(orig_rev, doc.rev) - self.assertFalse(doc.has_conflicts) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - self.assertGetDocConflicts(self.db, doc.doc_id, []) - - def test_resolve_doc_picks_biggest_vcr(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc2.rev, nested_doc), - (doc1.rev, simple_doc)]) - orig_doc1_rev = doc1.rev - self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) - self.assertFalse(doc1.has_conflicts) - self.assertNotEqual(orig_doc1_rev, doc1.rev) - self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) - self.assertGetDocConflicts(self.db, doc1.doc_id, []) - vcr_1 = vectorclock.VectorClockRev(orig_doc1_rev) - vcr_2 = vectorclock.VectorClockRev(doc2.rev) - vcr_new = vectorclock.VectorClockRev(doc1.rev) - self.assertTrue(vcr_new.is_newer(vcr_1)) - self.assertTrue(vcr_new.is_newer(vcr_2)) - - def test_resolve_doc_partial_not_winning(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc2.rev, nested_doc), - (doc1.rev, simple_doc)]) - content3 = '{"key": "valin3"}' - doc3 = self.make_document(doc1.doc_id, 'third:1', content3) - self.db._put_doc_if_newer( - doc3, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='bar') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc3.rev, content3), - (doc1.rev, simple_doc), - (doc2.rev, nested_doc)]) - self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) - self.assertTrue(doc1.has_conflicts) - self.assertGetDoc(self.db, doc1.doc_id, doc3.rev, content3, True) - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc3.rev, content3), - (doc1.rev, simple_doc)]) - - def test_resolve_doc_partial_winning(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - content3 = '{"key": "valin3"}' - doc3 = self.make_document(doc1.doc_id, 'third:1', content3) - self.db._put_doc_if_newer( - doc3, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='bar') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc3.rev, content3), - (doc1.rev, simple_doc), - (doc2.rev, nested_doc)]) - self.db.resolve_doc(doc1, [doc3.rev, doc1.rev]) - self.assertTrue(doc1.has_conflicts) - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc1.rev, simple_doc), - (doc2.rev, nested_doc)]) - - def test_resolve_doc_with_delete_conflict(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc1) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc2.rev, nested_doc), - (doc1.rev, None)]) - self.db.resolve_doc(doc2, [doc1.rev, doc2.rev]) - self.assertGetDocConflicts(self.db, doc1.doc_id, []) - self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, False) - - def test_resolve_doc_with_delete_to_delete(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc1) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc2.rev, nested_doc), - (doc1.rev, None)]) - self.db.resolve_doc(doc1, [doc1.rev, doc2.rev]) - self.assertGetDocConflicts(self.db, doc1.doc_id, []) - self.assertGetDocIncludeDeleted( - self.db, doc1.doc_id, doc1.rev, None, False) - - def test_put_doc_if_newer_save_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - # Document is inserted as a conflict - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - state, _ = self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual('conflicted', state) - # The database was updated - self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, nested_doc, True) - - def test_force_doc_conflict_supersedes_properly(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', '{"b": 1}') - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - doc3 = self.make_document(doc1.doc_id, 'altalt:1', '{"c": 1}') - self.db._put_doc_if_newer( - doc3, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='bar') - doc22 = self.make_document(doc1.doc_id, 'alternate:2', '{"b": 2}') - self.db._put_doc_if_newer( - doc22, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='zed') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [('alternate:2', doc22.get_json()), - ('altalt:1', doc3.get_json()), - (doc1.rev, simple_doc)]) - - def test_put_doc_if_newer_save_conflict_was_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc1) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertTrue(doc2.has_conflicts) - self.assertGetDoc( - self.db, doc1.doc_id, 'alternate:1', nested_doc, True) - self.assertGetDocConflicts(self.db, doc1.doc_id, - [('alternate:1', nested_doc), - (doc1.rev, None)]) - - def test_put_doc_if_newer_propagates_full_resolution(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - resolved_vcr = vectorclock.VectorClockRev(doc1.rev) - vcr_2 = vectorclock.VectorClockRev(doc2.rev) - resolved_vcr.maximize(vcr_2) - resolved_vcr.increment('alternate') - doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), - '{"good": 1}') - state, _ = self.db._put_doc_if_newer( - doc_resolved, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertEqual('inserted', state) - self.assertFalse(doc_resolved.has_conflicts) - self.assertGetDocConflicts(self.db, doc1.doc_id, []) - doc3 = self.db.get_doc(doc1.doc_id) - self.assertFalse(doc3.has_conflicts) - - def test_put_doc_if_newer_propagates_partial_resolution(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'altalt:1', '{}') - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - doc3 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc3, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [('alternate:1', nested_doc), - ('test:1', simple_doc), - ('altalt:1', '{}')]) - resolved_vcr = vectorclock.VectorClockRev(doc1.rev) - vcr_3 = vectorclock.VectorClockRev(doc3.rev) - resolved_vcr.maximize(vcr_3) - resolved_vcr.increment('alternate') - doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), - '{"good": 1}') - state, _ = self.db._put_doc_if_newer( - doc_resolved, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='foo3') - self.assertEqual('inserted', state) - self.assertTrue(doc_resolved.has_conflicts) - doc4 = self.db.get_doc(doc1.doc_id) - self.assertTrue(doc4.has_conflicts) - self.assertGetDocConflicts(self.db, doc1.doc_id, - [('alternate:2|test:1', '{"good": 1}'), - ('altalt:1', '{}')]) - - def test_put_doc_if_newer_replica_uid(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db._set_replica_gen_and_trans_id('other', 1, 'T-id') - doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', - nested_doc) - self.db._put_doc_if_newer(doc2, save_conflict=True, - replica_uid='other', replica_gen=2, - replica_trans_id='T-id2') - # Conflict vs the current update - doc2 = self.make_document(doc1.doc_id, doc1.rev + '|third:3', - '{}') - self.assertEqual('conflicted', - self.db._put_doc_if_newer( - doc2, - save_conflict=True, - replica_uid='other', - replica_gen=3, - replica_trans_id='T-id3')[0]) - self.assertEqual( - (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) - - def test_put_doc_if_newer_autoresolve_2(self): - # this is an ordering variant of _3, but that already works - # adding the test explicitly to catch the regression easily - doc_a1 = self.db.create_doc_from_json(simple_doc) - doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', "{}") - doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', - '{"a":"42"}') - doc_a3 = self.make_document(doc_a1.doc_id, 'test:2|other:1', "{}") - state, _ = self.db._put_doc_if_newer( - doc_a2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(state, 'inserted') - state, _ = self.db._put_doc_if_newer( - doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertEqual(state, 'conflicted') - state, _ = self.db._put_doc_if_newer( - doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='foo3') - self.assertEqual(state, 'inserted') - self.assertFalse(self.db.get_doc(doc_a1.doc_id).has_conflicts) - - def test_put_doc_if_newer_autoresolve_3(self): - doc_a1 = self.db.create_doc_from_json(simple_doc) - doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', "{}") - doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') - doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', "{}") - state, _ = self.db._put_doc_if_newer( - doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(state, 'inserted') - state, _ = self.db._put_doc_if_newer( - doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertEqual(state, 'conflicted') - state, _ = self.db._put_doc_if_newer( - doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='foo3') - self.assertEqual(state, 'superseded') - doc = self.db.get_doc(doc_a1.doc_id, True) - self.assertFalse(doc.has_conflicts) - rev = vectorclock.VectorClockRev(doc.rev) - rev_a3 = vectorclock.VectorClockRev('test:3') - rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') - self.assertTrue(rev.is_newer(rev_a3)) - self.assertTrue('test:4' in doc.rev) # locally increased - self.assertTrue(rev.is_newer(rev_a1b1)) - - def test_put_doc_if_newer_autoresolve_4(self): - doc_a1 = self.db.create_doc_from_json(simple_doc) - doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', None) - doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') - doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', None) - state, _ = self.db._put_doc_if_newer( - doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(state, 'inserted') - state, _ = self.db._put_doc_if_newer( - doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertEqual(state, 'conflicted') - state, _ = self.db._put_doc_if_newer( - doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='foo3') - self.assertEqual(state, 'superseded') - doc = self.db.get_doc(doc_a1.doc_id, True) - self.assertFalse(doc.has_conflicts) - rev = vectorclock.VectorClockRev(doc.rev) - rev_a3 = vectorclock.VectorClockRev('test:3') - rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') - self.assertTrue(rev.is_newer(rev_a3)) - self.assertTrue('test:4' in doc.rev) # locally increased - self.assertTrue(rev.is_newer(rev_a1b1)) - - def test_put_refuses_to_update_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - content2 = '{"key": "altval"}' - doc2 = self.make_document(doc1.doc_id, 'altrev:1', content2) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, content2, True) - content3 = '{"key": "local"}' - doc2.set_json(content3) - self.assertRaises(errors.ConflictedDoc, self.db.put_doc, doc2) - - def test_delete_refuses_for_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'altrev:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, True) - self.assertRaises(errors.ConflictedDoc, self.db.delete_doc, doc2) - - -@skip("Skiping tests imported from U1DB.") -class DatabaseIndexTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def assertParseError(self, definition): - self.db.create_doc_from_json(nested_doc) - self.assertRaises( - errors.IndexDefinitionParseError, self.db.create_index, 'idx', - definition) - - def assertIndexCreatable(self, definition): - name = "idx" - self.db.create_doc_from_json(nested_doc) - self.db.create_index(name, definition) - self.assertEqual( - [(name, [definition])], self.db.list_indexes()) - - def test_create_index(self): - self.db.create_index('test-idx', 'name') - self.assertEqual([('test-idx', ['name'])], - self.db.list_indexes()) - - def test_create_index_on_non_ascii_field_name(self): - doc = self.db.create_doc_from_json(json.dumps({u'\xe5': 'value'})) - self.db.create_index('test-idx', u'\xe5') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_list_indexes_with_non_ascii_field_names(self): - self.db.create_index('test-idx', u'\xe5') - self.assertEqual( - [('test-idx', [u'\xe5'])], self.db.list_indexes()) - - def test_create_index_evaluates_it(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_wildcard_matches_unicode_value(self): - doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) - self.db.create_index('test-idx', 'key') - self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) - - def test_retrieve_unicode_value_from_index(self): - doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc], self.db.get_from_index('test-idx', u"valu\xe5")) - - def test_create_index_fails_if_name_taken(self): - self.db.create_index('test-idx', 'key') - self.assertRaises(errors.IndexNameTakenError, - self.db.create_index, - 'test-idx', 'stuff') - - def test_create_index_does_not_fail_if_name_taken_with_same_index(self): - self.db.create_index('test-idx', 'key') - self.db.create_index('test-idx', 'key') - self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) - - def test_create_index_does_not_duplicate_indexed_fields(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.db.delete_index('test-idx') - self.db.create_index('test-idx', 'key') - self.assertEqual(1, len(self.db.get_from_index('test-idx', 'value'))) - - def test_delete_index_does_not_remove_fields_from_other_indexes(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.db.create_index('test-idx2', 'key') - self.db.delete_index('test-idx') - self.assertEqual(1, len(self.db.get_from_index('test-idx2', 'value'))) - - def test_create_index_after_deleting_document(self): - doc = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc2) - self.db.create_index('test-idx', 'key') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_delete_index(self): - self.db.create_index('test-idx', 'key') - self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) - self.db.delete_index('test-idx') - self.assertEqual([], self.db.list_indexes()) - - def test_create_adds_to_index(self): - self.db.create_index('test-idx', 'key') - doc = self.db.create_doc_from_json(simple_doc) - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_get_from_index_unmatched(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual([], self.db.get_from_index('test-idx', 'novalue')) - - def test_create_index_multiple_exact_matches(self): - doc = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual( - sorted([doc, doc2]), - sorted(self.db.get_from_index('test-idx', 'value'))) - - def test_get_from_index(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_get_from_index_multi(self): - content = '{"key": "value", "key2": "value2"}' - doc = self.db.create_doc_from_json(content) - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'value', 'value2')) - - def test_get_from_index_multi_list(self): - doc = self.db.create_doc_from_json( - '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'value', 'value2-1')) - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'value', 'value2-2')) - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'value', 'value2-3')) - self.assertEqual( - [('value', 'value2-1'), ('value', 'value2-2'), - ('value', 'value2-3')], - sorted(self.db.get_index_keys('test-idx'))) - - def test_get_from_index_sees_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key', 'key2') - alt_doc = self.make_document( - doc.doc_id, 'alternate:1', - '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - docs = self.db.get_from_index('test-idx', 'value', 'value2-1') - self.assertTrue(docs[0].has_conflicts) - - def test_get_index_keys_multi_list_list(self): - self.db.create_doc_from_json( - '{"key": "value1-1 value1-2 value1-3", ' - '"key2": ["value2-1", "value2-2", "value2-3"]}') - self.db.create_index('test-idx', 'split_words(key)', 'key2') - self.assertEqual( - [(u'value1-1', u'value2-1'), (u'value1-1', u'value2-2'), - (u'value1-1', u'value2-3'), (u'value1-2', u'value2-1'), - (u'value1-2', u'value2-2'), (u'value1-2', u'value2-3'), - (u'value1-3', u'value2-1'), (u'value1-3', u'value2-2'), - (u'value1-3', u'value2-3')], - sorted(self.db.get_index_keys('test-idx'))) - - def test_get_from_index_multi_ordered(self): - doc1 = self.db.create_doc_from_json( - '{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value3"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - doc4 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc4, doc3, doc2, doc1], - self.db.get_from_index('test-idx', 'v*', '*')) - - def test_get_range_from_index_start_end(self): - doc1 = self.db.create_doc_from_json('{"key": "value3"}') - doc2 = self.db.create_doc_from_json('{"key": "value2"}') - self.db.create_doc_from_json('{"key": "value4"}') - self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc2, doc1], - self.db.get_range_from_index('test-idx', 'value2', 'value3')) - - def test_get_range_from_index_start(self): - doc1 = self.db.create_doc_from_json('{"key": "value3"}') - doc2 = self.db.create_doc_from_json('{"key": "value2"}') - doc3 = self.db.create_doc_from_json('{"key": "value4"}') - self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc2, doc1, doc3], - self.db.get_range_from_index('test-idx', 'value2')) - - def test_get_range_from_index_sees_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - alt_doc = self.make_document( - doc.doc_id, 'alternate:1', '{"key": "valuedepalue"}') - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - docs = self.db.get_range_from_index('test-idx', 'a') - self.assertTrue(docs[0].has_conflicts) - - def test_get_range_from_index_end(self): - self.db.create_doc_from_json('{"key": "value3"}') - doc2 = self.db.create_doc_from_json('{"key": "value2"}') - self.db.create_doc_from_json('{"key": "value4"}') - doc4 = self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc4, doc2], - self.db.get_range_from_index('test-idx', None, 'value2')) - - def test_get_wildcard_range_from_index_start(self): - doc1 = self.db.create_doc_from_json('{"key": "value4"}') - doc2 = self.db.create_doc_from_json('{"key": "value23"}') - doc3 = self.db.create_doc_from_json('{"key": "value2"}') - doc4 = self.db.create_doc_from_json('{"key": "value22"}') - self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc3, doc4, doc2, doc1], - self.db.get_range_from_index('test-idx', 'value2*')) - - def test_get_wildcard_range_from_index_end(self): - self.db.create_doc_from_json('{"key": "value4"}') - doc2 = self.db.create_doc_from_json('{"key": "value23"}') - doc3 = self.db.create_doc_from_json('{"key": "value2"}') - doc4 = self.db.create_doc_from_json('{"key": "value22"}') - doc5 = self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc5, doc3, doc4, doc2], - self.db.get_range_from_index('test-idx', None, 'value2*')) - - def test_get_wildcard_range_from_index_start_end(self): - self.db.create_doc_from_json('{"key": "a"}') - self.db.create_doc_from_json('{"key": "boo3"}') - doc3 = self.db.create_doc_from_json('{"key": "catalyst"}') - doc4 = self.db.create_doc_from_json('{"key": "whaever"}') - self.db.create_doc_from_json('{"key": "zerg"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc3, doc4], - self.db.get_range_from_index('test-idx', 'cat*', 'zap*')) - - def test_get_range_from_index_multi_column_start_end(self): - self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value3"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc3, doc2], - self.db.get_range_from_index( - 'test-idx', ('value2', 'value2'), ('value2', 'value3'))) - - def test_get_range_from_index_multi_column_start(self): - doc1 = self.db.create_doc_from_json( - '{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value3"}') - self.db.create_doc_from_json('{"key": "value2", "key2": "value2"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc2, doc1], - self.db.get_range_from_index('test-idx', ('value2', 'value3'))) - - def test_get_range_from_index_multi_column_end(self): - self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value3"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - doc4 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc4, doc3, doc2], - self.db.get_range_from_index( - 'test-idx', None, ('value2', 'value3'))) - - def test_get_wildcard_range_from_index_multi_column_start(self): - doc1 = self.db.create_doc_from_json( - '{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value23"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc3, doc2, doc1], - self.db.get_range_from_index('test-idx', ('value2', 'value2*'))) - - def test_get_wildcard_range_from_index_multi_column_end(self): - self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value23"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - doc4 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc4, doc3, doc2], - self.db.get_range_from_index( - 'test-idx', None, ('value2', 'value2*'))) - - def test_get_glob_range_from_index_multi_column_start(self): - doc1 = self.db.create_doc_from_json( - '{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value23"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value2"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc2, doc1], - self.db.get_range_from_index('test-idx', ('value2', '*'))) - - def test_get_glob_range_from_index_multi_column_end(self): - self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value23"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value2"}') - doc4 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc4, doc3, doc2], - self.db.get_range_from_index('test-idx', None, ('value2', '*'))) - - def test_get_range_from_index_illegal_wildcard_order(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_range_from_index, 'test-idx', ('*', 'v2')) - - def test_get_range_from_index_illegal_glob_after_wildcard(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_range_from_index, 'test-idx', ('*', 'v*')) - - def test_get_range_from_index_illegal_wildcard_order_end(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_range_from_index, 'test-idx', None, ('*', 'v2')) - - def test_get_range_from_index_illegal_glob_after_wildcard_end(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_range_from_index, 'test-idx', None, ('*', 'v*')) - - def test_get_from_index_fails_if_no_index(self): - self.assertRaises( - errors.IndexDoesNotExist, self.db.get_from_index, 'foo') - - def test_get_index_keys_fails_if_no_index(self): - self.assertRaises(errors.IndexDoesNotExist, - self.db.get_index_keys, - 'foo') - - def test_get_index_keys_works_if_no_docs(self): - self.db.create_index('test-idx', 'key') - self.assertEqual([], self.db.get_index_keys('test-idx')) - - def test_put_updates_index(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - new_content = '{"key": "altval"}' - doc.set_json(new_content) - self.db.put_doc(doc) - self.assertEqual([], self.db.get_from_index('test-idx', 'value')) - self.assertEqual([doc], self.db.get_from_index('test-idx', 'altval')) - - def test_delete_updates_index(self): - doc = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual( - sorted([doc, doc2]), - sorted(self.db.get_from_index('test-idx', 'value'))) - self.db.delete_doc(doc) - self.assertEqual([doc2], self.db.get_from_index('test-idx', 'value')) - - def test_get_from_index_illegal_number_of_entries(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidValueForIndex, self.db.get_from_index, 'test-idx') - self.assertRaises( - errors.InvalidValueForIndex, - self.db.get_from_index, 'test-idx', 'v1') - self.assertRaises( - errors.InvalidValueForIndex, - self.db.get_from_index, 'test-idx', 'v1', 'v2', 'v3') - - def test_get_from_index_illegal_wildcard_order(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_from_index, 'test-idx', '*', 'v2') - - def test_get_from_index_illegal_glob_after_wildcard(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_from_index, 'test-idx', '*', 'v*') - - def test_get_all_from_index(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - # This one should not be in the index - self.db.create_doc_from_json('{"no": "key"}') - diff_value_doc = '{"key": "diff value"}' - doc4 = self.db.create_doc_from_json(diff_value_doc) - # This is essentially a 'prefix' match, but we match every entry. - self.assertEqual( - sorted([doc1, doc2, doc4]), - sorted(self.db.get_from_index('test-idx', '*'))) - - def test_get_all_from_index_ordered(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json('{"key": "value x"}') - doc2 = self.db.create_doc_from_json('{"key": "value b"}') - doc3 = self.db.create_doc_from_json('{"key": "value a"}') - doc4 = self.db.create_doc_from_json('{"key": "value m"}') - # This is essentially a 'prefix' match, but we match every entry. - self.assertEqual( - [doc3, doc2, doc4, doc1], self.db.get_from_index('test-idx', '*')) - - def test_put_updates_when_adding_key(self): - doc = self.db.create_doc_from_json("{}") - self.db.create_index('test-idx', 'key') - self.assertEqual([], self.db.get_from_index('test-idx', '*')) - doc.set_json(simple_doc) - self.db.put_doc(doc) - self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) - - def test_get_from_index_empty_string(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json(simple_doc) - content2 = '{"key": ""}' - doc2 = self.db.create_doc_from_json(content2) - self.assertEqual([doc2], self.db.get_from_index('test-idx', '')) - # Empty string matches the wildcard. - self.assertEqual( - sorted([doc1, doc2]), - sorted(self.db.get_from_index('test-idx', '*'))) - - def test_get_from_index_not_null(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json(simple_doc) - self.db.create_doc_from_json('{"key": null}') - self.assertEqual([doc1], self.db.get_from_index('test-idx', '*')) - - def test_get_partial_from_index(self): - content1 = '{"k1": "v1", "k2": "v2"}' - content2 = '{"k1": "v1", "k2": "x2"}' - content3 = '{"k1": "v1", "k2": "y2"}' - # doc4 has a different k1 value, so it doesn't match the prefix. - content4 = '{"k1": "NN", "k2": "v2"}' - doc1 = self.db.create_doc_from_json(content1) - doc2 = self.db.create_doc_from_json(content2) - doc3 = self.db.create_doc_from_json(content3) - self.db.create_doc_from_json(content4) - self.db.create_index('test-idx', 'k1', 'k2') - self.assertEqual( - sorted([doc1, doc2, doc3]), - sorted(self.db.get_from_index('test-idx', "v1", "*"))) - - def test_get_glob_match(self): - # Note: the exact glob syntax is probably subject to change - content1 = '{"k1": "v1", "k2": "v1"}' - content2 = '{"k1": "v1", "k2": "v2"}' - content3 = '{"k1": "v1", "k2": "v3"}' - # doc4 has a different k2 prefix value, so it doesn't match - content4 = '{"k1": "v1", "k2": "ZZ"}' - self.db.create_index('test-idx', 'k1', 'k2') - doc1 = self.db.create_doc_from_json(content1) - doc2 = self.db.create_doc_from_json(content2) - doc3 = self.db.create_doc_from_json(content3) - self.db.create_doc_from_json(content4) - self.assertEqual( - sorted([doc1, doc2, doc3]), - sorted(self.db.get_from_index('test-idx', "v1", "v*"))) - - def test_nested_index(self): - doc = self.db.create_doc_from_json(nested_doc) - self.db.create_index('test-idx', 'sub.doc') - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'underneath')) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertEqual( - sorted([doc, doc2]), - sorted(self.db.get_from_index('test-idx', 'underneath'))) - - def test_nested_nonexistent(self): - self.db.create_doc_from_json(nested_doc) - # sub exists, but sub.foo does not: - self.db.create_index('test-idx', 'sub.foo') - self.assertEqual([], self.db.get_from_index('test-idx', '*')) - - def test_nested_nonexistent2(self): - self.db.create_doc_from_json(nested_doc) - self.db.create_index('test-idx', 'sub.foo.bar.baz.qux.fnord') - self.assertEqual([], self.db.get_from_index('test-idx', '*')) - - def test_nested_traverses_lists(self): - # subpath finds dicts in list - doc = self.db.create_doc_from_json( - '{"foo": [{"zap": "bar"}, {"zap": "baz"}]}') - # subpath only finds dicts in list - self.db.create_doc_from_json('{"foo": ["zap", "baz"]}') - self.db.create_index('test-idx', 'foo.zap') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'bar')) - self.assertEqual([doc], self.db.get_from_index('test-idx', 'baz')) - - def test_nested_list_traversal(self): - # subpath finds dicts in list - doc = self.db.create_doc_from_json( - '{"foo": [{"zap": [{"qux": "fnord"}, {"qux": "zombo"}]},' - '{"zap": "baz"}]}') - # subpath only finds dicts in list - self.db.create_index('test-idx', 'foo.zap.qux') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'fnord')) - self.assertEqual([doc], self.db.get_from_index('test-idx', 'zombo')) - - def test_index_list1(self): - self.db.create_index("index", "name") - content = '{"name": ["foo", "bar"]}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "bar") - self.assertEqual([doc], rows) - - def test_index_list2(self): - self.db.create_index("index", "name") - content = '{"name": ["foo", "bar"]}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_get_from_index_case_sensitive(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json(simple_doc) - self.assertEqual([], self.db.get_from_index('test-idx', 'V*')) - self.assertEqual([doc1], self.db.get_from_index('test-idx', 'v*')) - - def test_get_from_index_illegal_glob_before_value(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_from_index, 'test-idx', 'v*', 'v2') - - def test_get_from_index_illegal_glob_after_glob(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_from_index, 'test-idx', 'v*', 'v*') - - def test_get_from_index_with_sql_wildcards(self): - self.db.create_index('test-idx', 'key') - content1 = '{"key": "va%lue"}' - content2 = '{"key": "value"}' - content3 = '{"key": "va_lue"}' - doc1 = self.db.create_doc_from_json(content1) - self.db.create_doc_from_json(content2) - doc3 = self.db.create_doc_from_json(content3) - # The '%' in the search should be treated literally, not as a sql - # globbing character. - self.assertEqual([doc1], self.db.get_from_index('test-idx', 'va%*')) - # Same for '_' - self.assertEqual([doc3], self.db.get_from_index('test-idx', 'va_*')) - - def test_get_from_index_with_lower(self): - self.db.create_index("index", "lower(name)") - content = '{"name": "Foo"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_get_from_index_with_lower_matches_same_case(self): - self.db.create_index("index", "lower(name)") - content = '{"name": "foo"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_index_lower_doesnt_match_different_case(self): - self.db.create_index("index", "lower(name)") - content = '{"name": "Foo"}' - self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "Foo") - self.assertEqual([], rows) - - def test_index_lower_doesnt_match_other_index(self): - self.db.create_index("index", "lower(name)") - self.db.create_index("other_index", "name") - content = '{"name": "Foo"}' - self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "Foo") - self.assertEqual(0, len(rows)) - - def test_index_split_words_match_first(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo bar"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_index_split_words_match_second(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo bar"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "bar") - self.assertEqual([doc], rows) - - def test_index_split_words_match_both(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo foo"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_index_split_words_double_space(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo bar"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "bar") - self.assertEqual([doc], rows) - - def test_index_split_words_leading_space(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": " foo bar"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_index_split_words_trailing_space(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo bar "}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "bar") - self.assertEqual([doc], rows) - - def test_get_from_index_with_number(self): - self.db.create_index("index", "number(foo, 5)") - content = '{"foo": 12}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "00012") - self.assertEqual([doc], rows) - - def test_get_from_index_with_number_bigger_than_padding(self): - self.db.create_index("index", "number(foo, 5)") - content = '{"foo": 123456}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "123456") - self.assertEqual([doc], rows) - - def test_number_mapping_ignores_non_numbers(self): - self.db.create_index("index", "number(foo, 5)") - content = '{"foo": 56}' - doc1 = self.db.create_doc_from_json(content) - content = '{"foo": "this is not a maigret painting"}' - self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "*") - self.assertEqual([doc1], rows) - - def test_get_from_index_with_bool(self): - self.db.create_index("index", "bool(foo)") - content = '{"foo": true}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "1") - self.assertEqual([doc], rows) - - def test_get_from_index_with_bool_false(self): - self.db.create_index("index", "bool(foo)") - content = '{"foo": false}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "0") - self.assertEqual([doc], rows) - - def test_get_from_index_with_non_bool(self): - self.db.create_index("index", "bool(foo)") - content = '{"foo": 42}' - self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "*") - self.assertEqual([], rows) - - def test_get_from_index_with_combine(self): - self.db.create_index("index", "combine(foo, bar)") - content = '{"foo": "value1", "bar": "value2"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "value1") - self.assertEqual([doc], rows) - rows = self.db.get_from_index("index", "value2") - self.assertEqual([doc], rows) - - def test_get_complex_combine(self): - self.db.create_index( - "index", "combine(number(foo, 5), lower(bar), split_words(baz))") - content = '{"foo": 12, "bar": "ALLCAPS", "baz": "qux nox"}' - doc = self.db.create_doc_from_json(content) - content = '{"foo": "not a number", "bar": "something"}' - doc2 = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "00012") - self.assertEqual([doc], rows) - rows = self.db.get_from_index("index", "allcaps") - self.assertEqual([doc], rows) - rows = self.db.get_from_index("index", "nox") - self.assertEqual([doc], rows) - rows = self.db.get_from_index("index", "something") - self.assertEqual([doc2], rows) - - def test_get_index_keys_from_index(self): - self.db.create_index('test-idx', 'key') - content1 = '{"key": "value1"}' - content2 = '{"key": "value2"}' - content3 = '{"key": "value2"}' - self.db.create_doc_from_json(content1) - self.db.create_doc_from_json(content2) - self.db.create_doc_from_json(content3) - self.assertEqual( - [('value1',), ('value2',)], - sorted(self.db.get_index_keys('test-idx'))) - - def test_get_index_keys_from_multicolumn_index(self): - self.db.create_index('test-idx', 'key1', 'key2') - content1 = '{"key1": "value1", "key2": "val2-1"}' - content2 = '{"key1": "value2", "key2": "val2-2"}' - content3 = '{"key1": "value2", "key2": "val2-2"}' - content4 = '{"key1": "value2", "key2": "val3"}' - self.db.create_doc_from_json(content1) - self.db.create_doc_from_json(content2) - self.db.create_doc_from_json(content3) - self.db.create_doc_from_json(content4) - self.assertEqual([ - ('value1', 'val2-1'), - ('value2', 'val2-2'), - ('value2', 'val3')], - sorted(self.db.get_index_keys('test-idx'))) - - def test_empty_expr(self): - self.assertParseError('') - - def test_nested_unknown_operation(self): - self.assertParseError('unknown_operation(field1)') - - def test_parse_missing_close_paren(self): - self.assertParseError("lower(a") - - def test_parse_trailing_close_paren(self): - self.assertParseError("lower(ab))") - - def test_parse_trailing_chars(self): - self.assertParseError("lower(ab)adsf") - - def test_parse_empty_op(self): - self.assertParseError("(ab)") - - def test_parse_top_level_commas(self): - self.assertParseError("a, b") - - def test_invalid_field_name(self): - self.assertParseError("a.") - - def test_invalid_inner_field_name(self): - self.assertParseError("lower(a.)") - - def test_gobbledigook(self): - self.assertParseError("(@#@cc @#!*DFJSXV(()jccd") - - def test_leading_space(self): - self.assertIndexCreatable(" lower(a)") - - def test_trailing_space(self): - self.assertIndexCreatable("lower(a) ") - - def test_spaces_before_open_paren(self): - self.assertIndexCreatable("lower (a)") - - def test_spaces_after_open_paren(self): - self.assertIndexCreatable("lower( a)") - - def test_spaces_before_close_paren(self): - self.assertIndexCreatable("lower(a )") - - def test_spaces_before_comma(self): - self.assertIndexCreatable("combine(a , b , c)") - - def test_spaces_after_comma(self): - self.assertIndexCreatable("combine(a, b, c)") - - def test_all_together_now(self): - self.assertParseError(' (a) ') - - def test_all_together_now2(self): - self.assertParseError('combine(lower(x)x,foo)') - - -@skip("Skiping tests imported from U1DB.") -class PythonBackendTests(tests.DatabaseBaseTests): - - def setUp(self): - super(PythonBackendTests, self).setUp() - self.simple_doc = json.loads(simple_doc) - - def test_create_doc_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') - self.assertTrue(isinstance(doc, TestAlternativeDocument)) - - def test_get_doc_after_put_with_factory(self): - doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') - self.db.set_document_factory(TestAlternativeDocument) - result = self.db.get_doc('my_doc_id') - self.assertTrue(isinstance(result, TestAlternativeDocument)) - self.assertEqual(doc.doc_id, result.doc_id) - self.assertEqual(doc.rev, result.rev) - self.assertEqual(doc.get_json(), result.get_json()) - self.assertEqual(False, result.has_conflicts) - - def test_get_doc_nonexisting_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - self.assertIs(None, self.db.get_doc('non-existing')) - - def test_get_all_docs_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - self.db.create_doc(self.simple_doc) - self.assertTrue(isinstance( - list(self.db.get_all_docs()[1])[0], TestAlternativeDocument)) - - def test_get_docs_conflicted_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - doc1 = self.db.create_doc(self.simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertTrue( - isinstance( - list(self.db.get_docs([doc1.doc_id]))[0], - TestAlternativeDocument)) - - def test_get_from_index_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - self.db.create_doc(self.simple_doc) - self.db.create_index('test-idx', 'key') - self.assertTrue( - isinstance( - self.db.get_from_index('test-idx', 'value')[0], - TestAlternativeDocument)) - - def test_sync_exchange_updates_indexes(self): - doc = self.db.create_doc(self.simple_doc) - self.db.create_index('test-idx', 'key') - new_content = '{"key": "altval"}' - other_rev = 'test:1|z:2' - st = self.db.get_sync_target() - - def ignore(doc_id, doc_rev, doc): - pass - - doc_other = self.make_document(doc.doc_id, other_rev, new_content) - docs_by_gen = [(doc_other, 10, 'T-sid')] - st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=ignore) - self.assertGetDoc(self.db, doc.doc_id, other_rev, new_content, False) - self.assertEqual( - [doc_other], self.db.get_from_index('test-idx', 'altval')) - self.assertEqual([], self.db.get_from_index('test-idx', 'value')) - - -# Use a custom loader to apply the scenarios at load time. -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py deleted file mode 100644 index 23502b4b..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - - -from unittest import skip -from u1db import errors - -from leap.soledad.common.tests import u1db_tests as tests - - -@skip("Skiping tests imported from U1DB.") -class TestDocument(tests.TestCase): - - scenarios = ([( - 'py', {'make_document_for_test': tests.make_document_for_test})]) # + - # tests.C_DATABASE_SCENARIOS) - - def test_create_doc(self): - doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) - self.assertEqual('doc-id', doc.doc_id) - self.assertEqual('uid:1', doc.rev) - self.assertEqual(tests.simple_doc, doc.get_json()) - self.assertFalse(doc.has_conflicts) - - def test__repr__(self): - doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) - self.assertEqual( - '%s(doc-id, uid:1, \'{"key": "value"}\')' - % (doc.__class__.__name__,), - repr(doc)) - - def test__repr__conflicted(self): - doc = self.make_document('doc-id', 'uid:1', tests.simple_doc, - has_conflicts=True) - self.assertEqual( - '%s(doc-id, uid:1, conflicted, \'{"key": "value"}\')' - % (doc.__class__.__name__,), - repr(doc)) - - def test__lt__(self): - doc_a = self.make_document('a', 'b', '{}') - doc_b = self.make_document('b', 'b', '{}') - self.assertTrue(doc_a < doc_b) - self.assertTrue(doc_b > doc_a) - doc_aa = self.make_document('a', 'a', '{}') - self.assertTrue(doc_aa < doc_a) - - def test__eq__(self): - doc_a = self.make_document('a', 'b', '{}') - doc_b = self.make_document('a', 'b', '{}') - self.assertTrue(doc_a == doc_b) - doc_b = self.make_document('a', 'b', '{}', has_conflicts=True) - self.assertFalse(doc_a == doc_b) - - def test_non_json_dict(self): - self.assertRaises( - errors.InvalidJSON, self.make_document, 'id', 'uid:1', - '"not a json dictionary"') - - def test_non_json(self): - self.assertRaises( - errors.InvalidJSON, self.make_document, 'id', 'uid:1', - 'not a json dictionary') - - def test_get_size(self): - doc_a = self.make_document('a', 'b', '{"some": "content"}') - self.assertEqual( - len('a' + 'b' + '{"some": "content"}'), doc_a.get_size()) - - def test_get_size_empty_document(self): - doc_a = self.make_document('a', 'b', None) - self.assertEqual(len('a' + 'b'), doc_a.get_size()) - - -@skip("Skiping tests imported from U1DB.") -class TestPyDocument(tests.TestCase): - - scenarios = ([( - 'py', {'make_document_for_test': tests.make_document_for_test})]) - - def test_get_content(self): - doc = self.make_document('id', 'rev', '{"content":""}') - self.assertEqual({"content": ""}, doc.content) - doc.set_json('{"content": "new"}') - self.assertEqual({"content": "new"}, doc.content) - - def test_set_content(self): - doc = self.make_document('id', 'rev', '{"content":""}') - doc.content = {"content": "new"} - self.assertEqual('{"content": "new"}', doc.get_json()) - - def test_set_bad_content(self): - doc = self.make_document('id', 'rev', '{"content":""}') - self.assertRaises( - errors.InvalidContent, setattr, doc, 'content', - '{"content": "new"}') - - def test_is_tombstone(self): - doc_a = self.make_document('a', 'b', '{}') - self.assertFalse(doc_a.is_tombstone()) - doc_a.set_json(None) - self.assertTrue(doc_a.is_tombstone()) - - def test_make_tombstone(self): - doc_a = self.make_document('a', 'b', '{}') - self.assertFalse(doc_a.is_tombstone()) - doc_a.make_tombstone() - self.assertTrue(doc_a.is_tombstone()) - - def test_same_content_as(self): - doc_a = self.make_document('a', 'b', '{}') - doc_b = self.make_document('d', 'e', '{}') - self.assertTrue(doc_a.same_content_as(doc_b)) - doc_b = self.make_document('p', 'q', '{}', has_conflicts=True) - self.assertTrue(doc_a.same_content_as(doc_b)) - doc_b.content['key'] = 'value' - self.assertFalse(doc_a.same_content_as(doc_b)) - - def test_same_content_as_json_order(self): - doc_a = self.make_document( - 'a', 'b', '{"key1": "val1", "key2": "val2"}') - doc_b = self.make_document( - 'c', 'd', '{"key2": "val2", "key1": "val1"}') - self.assertTrue(doc_a.same_content_as(doc_b)) - - def test_set_json(self): - doc = self.make_document('id', 'rev', '{"content":""}') - doc.set_json('{"content": "new"}') - self.assertEqual('{"content": "new"}', doc.get_json()) - - def test_set_json_non_dict(self): - doc = self.make_document('id', 'rev', '{"content":""}') - self.assertRaises(errors.InvalidJSON, doc.set_json, '"is not a dict"') - - def test_set_json_error(self): - doc = self.make_document('id', 'rev', '{"content":""}') - self.assertRaises(errors.InvalidJSON, doc.set_json, 'is not json') - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py deleted file mode 100644 index 973c3b26..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py +++ /dev/null @@ -1,364 +0,0 @@ -# Copyright 2011-2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""Tests for HTTPDatabase""" - -from oauth import oauth -import json - -from u1db import ( - errors, -) - -from unittest import skip - -from leap.soledad.common.tests import u1db_tests as tests - -from u1db.remote import ( - http_client, -) - - -@skip("Skiping tests imported from U1DB.") -class TestEncoder(tests.TestCase): - - def test_encode_string(self): - self.assertEqual("foo", http_client._encode_query_parameter("foo")) - - def test_encode_true(self): - self.assertEqual("true", http_client._encode_query_parameter(True)) - - def test_encode_false(self): - self.assertEqual("false", http_client._encode_query_parameter(False)) - - -@skip("Skiping tests imported from U1DB.") -class TestHTTPClientBase(tests.TestCaseWithServer): - - def setUp(self): - super(TestHTTPClientBase, self).setUp() - self.errors = 0 - - def app(self, environ, start_response): - if environ['PATH_INFO'].endswith('echo'): - start_response("200 OK", [('Content-Type', 'application/json')]) - ret = {} - for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): - ret[name] = environ[name] - if environ['REQUEST_METHOD'] in ('PUT', 'POST'): - ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] - content_length = int(environ['CONTENT_LENGTH']) - ret['body'] = environ['wsgi.input'].read(content_length) - return [json.dumps(ret)] - elif environ['PATH_INFO'].endswith('error_then_accept'): - if self.errors >= 3: - start_response( - "200 OK", [('Content-Type', 'application/json')]) - ret = {} - for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): - ret[name] = environ[name] - if environ['REQUEST_METHOD'] in ('PUT', 'POST'): - ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] - content_length = int(environ['CONTENT_LENGTH']) - ret['body'] = '{"oki": "doki"}' - return [json.dumps(ret)] - self.errors += 1 - content_length = int(environ['CONTENT_LENGTH']) - error = json.loads( - environ['wsgi.input'].read(content_length)) - response = error['response'] - # In debug mode, wsgiref has an assertion that the status parameter - # is a 'str' object. However error['status'] returns a unicode - # object. - status = str(error['status']) - if isinstance(response, unicode): - response = str(response) - if isinstance(response, str): - start_response(status, [('Content-Type', 'text/plain')]) - return [str(response)] - else: - start_response(status, [('Content-Type', 'application/json')]) - return [json.dumps(response)] - elif environ['PATH_INFO'].endswith('error'): - self.errors += 1 - content_length = int(environ['CONTENT_LENGTH']) - error = json.loads( - environ['wsgi.input'].read(content_length)) - response = error['response'] - # In debug mode, wsgiref has an assertion that the status parameter - # is a 'str' object. However error['status'] returns a unicode - # object. - status = str(error['status']) - if isinstance(response, unicode): - response = str(response) - if isinstance(response, str): - start_response(status, [('Content-Type', 'text/plain')]) - return [str(response)] - else: - start_response(status, [('Content-Type', 'application/json')]) - return [json.dumps(response)] - elif '/oauth' in environ['PATH_INFO']: - base_url = self.getURL('').rstrip('/') - oauth_req = oauth.OAuthRequest.from_request( - http_method=environ['REQUEST_METHOD'], - http_url=base_url + environ['PATH_INFO'], - headers={'Authorization': environ['HTTP_AUTHORIZATION']}, - query_string=environ['QUERY_STRING'] - ) - oauth_server = oauth.OAuthServer(tests.testingOAuthStore) - oauth_server.add_signature_method(tests.sign_meth_HMAC_SHA1) - try: - consumer, token, params = oauth_server.verify_request( - oauth_req) - except oauth.OAuthError, e: - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - start_response("200 OK", [('Content-Type', 'application/json')]) - return [json.dumps([environ['PATH_INFO'], token.key, params])] - - def make_app(self): - return self.app - - def getClient(self, **kwds): - self.startServer() - return http_client.HTTPClientBase(self.getURL('dbase'), **kwds) - - def test_construct(self): - self.startServer() - url = self.getURL() - cli = http_client.HTTPClientBase(url) - self.assertEqual(url, cli._url.geturl()) - self.assertIs(None, cli._conn) - - def test_parse_url(self): - cli = http_client.HTTPClientBase( - '%s://127.0.0.1:12345/' % self.url_scheme) - self.assertEqual(self.url_scheme, cli._url.scheme) - self.assertEqual('127.0.0.1', cli._url.hostname) - self.assertEqual(12345, cli._url.port) - self.assertEqual('/', cli._url.path) - - def test__ensure_connection(self): - cli = self.getClient() - self.assertIs(None, cli._conn) - cli._ensure_connection() - self.assertIsNot(None, cli._conn) - conn = cli._conn - cli._ensure_connection() - self.assertIs(conn, cli._conn) - - def test_close(self): - cli = self.getClient() - cli._ensure_connection() - cli.close() - self.assertIs(None, cli._conn) - - def test__request(self): - cli = self.getClient() - res, headers = cli._request('PUT', ['echo'], {}, {}) - self.assertEqual({'CONTENT_TYPE': 'application/json', - 'PATH_INFO': '/dbase/echo', - 'QUERY_STRING': '', - 'body': '{}', - 'REQUEST_METHOD': 'PUT'}, json.loads(res)) - - res, headers = cli._request('GET', ['doc', 'echo'], {'a': 1}) - self.assertEqual({'PATH_INFO': '/dbase/doc/echo', - 'QUERY_STRING': 'a=1', - 'REQUEST_METHOD': 'GET'}, json.loads(res)) - - res, headers = cli._request('GET', ['doc', '%FFFF', 'echo'], {'a': 1}) - self.assertEqual({'PATH_INFO': '/dbase/doc/%FFFF/echo', - 'QUERY_STRING': 'a=1', - 'REQUEST_METHOD': 'GET'}, json.loads(res)) - - res, headers = cli._request('POST', ['echo'], {'b': 2}, 'Body', - 'application/x-test') - self.assertEqual({'CONTENT_TYPE': 'application/x-test', - 'PATH_INFO': '/dbase/echo', - 'QUERY_STRING': 'b=2', - 'body': 'Body', - 'REQUEST_METHOD': 'POST'}, json.loads(res)) - - def test__request_json(self): - cli = self.getClient() - res, headers = cli._request_json( - 'POST', ['echo'], {'b': 2}, {'a': 'x'}) - self.assertEqual('application/json', headers['content-type']) - self.assertEqual({'CONTENT_TYPE': 'application/json', - 'PATH_INFO': '/dbase/echo', - 'QUERY_STRING': 'b=2', - 'body': '{"a": "x"}', - 'REQUEST_METHOD': 'POST'}, res) - - def test_unspecified_http_error(self): - cli = self.getClient() - self.assertRaises(errors.HTTPError, - cli._request_json, 'POST', ['error'], {}, - {'status': "500 Internal Error", - 'response': "Crash."}) - try: - cli._request_json('POST', ['error'], {}, - {'status': "500 Internal Error", - 'response': "Fail."}) - except errors.HTTPError, e: - pass - - self.assertEqual(500, e.status) - self.assertEqual("Fail.", e.message) - self.assertTrue("content-type" in e.headers) - - def test_revision_conflict(self): - cli = self.getClient() - self.assertRaises(errors.RevisionConflict, - cli._request_json, 'POST', ['error'], {}, - {'status': "409 Conflict", - 'response': {"error": "revision conflict"}}) - - def test_unavailable_proper(self): - cli = self.getClient() - cli._delays = (0, 0, 0, 0, 0) - self.assertRaises(errors.Unavailable, - cli._request_json, 'POST', ['error'], {}, - {'status': "503 Service Unavailable", - 'response': {"error": "unavailable"}}) - self.assertEqual(5, self.errors) - - def test_unavailable_then_available(self): - cli = self.getClient() - cli._delays = (0, 0, 0, 0, 0) - res, headers = cli._request_json( - 'POST', ['error_then_accept'], {'b': 2}, - {'status': "503 Service Unavailable", - 'response': {"error": "unavailable"}}) - self.assertEqual('application/json', headers['content-type']) - self.assertEqual({'CONTENT_TYPE': 'application/json', - 'PATH_INFO': '/dbase/error_then_accept', - 'QUERY_STRING': 'b=2', - 'body': '{"oki": "doki"}', - 'REQUEST_METHOD': 'POST'}, res) - self.assertEqual(3, self.errors) - - def test_unavailable_random_source(self): - cli = self.getClient() - cli._delays = (0, 0, 0, 0, 0) - try: - cli._request_json('POST', ['error'], {}, - {'status': "503 Service Unavailable", - 'response': "random unavailable."}) - except errors.Unavailable, e: - pass - - self.assertEqual(503, e.status) - self.assertEqual("random unavailable.", e.message) - self.assertTrue("content-type" in e.headers) - self.assertEqual(5, self.errors) - - def test_document_too_big(self): - cli = self.getClient() - self.assertRaises(errors.DocumentTooBig, - cli._request_json, 'POST', ['error'], {}, - {'status': "403 Forbidden", - 'response': {"error": "document too big"}}) - - def test_user_quota_exceeded(self): - cli = self.getClient() - self.assertRaises(errors.UserQuotaExceeded, - cli._request_json, 'POST', ['error'], {}, - {'status': "403 Forbidden", - 'response': {"error": "user quota exceeded"}}) - - def test_user_needs_subscription(self): - cli = self.getClient() - self.assertRaises(errors.SubscriptionNeeded, - cli._request_json, 'POST', ['error'], {}, - {'status': "403 Forbidden", - 'response': {"error": "user needs subscription"}}) - - def test_generic_u1db_error(self): - cli = self.getClient() - self.assertRaises(errors.U1DBError, - cli._request_json, 'POST', ['error'], {}, - {'status': "400 Bad Request", - 'response': {"error": "error"}}) - try: - cli._request_json('POST', ['error'], {}, - {'status': "400 Bad Request", - 'response': {"error": "error"}}) - except errors.U1DBError, e: - pass - self.assertIs(e.__class__, errors.U1DBError) - - def test_unspecified_bad_request(self): - cli = self.getClient() - self.assertRaises(errors.HTTPError, - cli._request_json, 'POST', ['error'], {}, - {'status': "400 Bad Request", - 'response': "<Bad Request>"}) - try: - cli._request_json('POST', ['error'], {}, - {'status': "400 Bad Request", - 'response': "<Bad Request>"}) - except errors.HTTPError, e: - pass - - self.assertEqual(400, e.status) - self.assertEqual("<Bad Request>", e.message) - self.assertTrue("content-type" in e.headers) - - def test_oauth(self): - cli = self.getClient() - cli.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - params = {'x': u'\xf0', 'y': "foo"} - res, headers = cli._request('GET', ['doc', 'oauth'], params) - self.assertEqual( - ['/dbase/doc/oauth', tests.token1.key, params], json.loads(res)) - - # oauth does its own internal quoting - params = {'x': u'\xf0', 'y': "foo"} - res, headers = cli._request('GET', ['doc', 'oauth', 'foo bar'], params) - self.assertEqual( - ['/dbase/doc/oauth/foo bar', tests.token1.key, params], - json.loads(res)) - - def test_oauth_ctr_creds(self): - cli = self.getClient(creds={'oauth': { - 'consumer_key': tests.consumer1.key, - 'consumer_secret': tests.consumer1.secret, - 'token_key': tests.token1.key, - 'token_secret': tests.token1.secret, - }}) - params = {'x': u'\xf0', 'y': "foo"} - res, headers = cli._request('GET', ['doc', 'oauth'], params) - self.assertEqual( - ['/dbase/doc/oauth', tests.token1.key, params], json.loads(res)) - - def test_unknown_creds(self): - self.assertRaises(errors.UnknownAuthMethod, - self.getClient, creds={'foo': {}}) - self.assertRaises(errors.UnknownAuthMethod, - self.getClient, creds={}) - - def test_oauth_Unauthorized(self): - cli = self.getClient() - cli.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, "WRONG") - params = {'y': 'foo'} - self.assertRaises(errors.Unauthorized, cli._request, 'GET', - ['doc', 'oauth'], params) diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py deleted file mode 100644 index 015e6e69..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""Tests for HTTPDatabase""" - -import inspect -import json - -from unittest import skip - -from u1db import errors -from u1db import Document -from u1db.remote import http_database -from u1db.remote import http_target - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import make_http_app - - -@skip("Skiping tests imported from U1DB.") -class TestHTTPDatabaseSimpleOperations(tests.TestCase): - - def setUp(self): - super(TestHTTPDatabaseSimpleOperations, self).setUp() - self.db = http_database.HTTPDatabase('dbase') - self.db._conn = object() # crash if used - self.got = None - self.response_val = None - - def _request(method, url_parts, params=None, body=None, - content_type=None): - self.got = method, url_parts, params, body, content_type - if isinstance(self.response_val, Exception): - raise self.response_val - return self.response_val - - def _request_json(method, url_parts, params=None, body=None, - content_type=None): - self.got = method, url_parts, params, body, content_type - if isinstance(self.response_val, Exception): - raise self.response_val - return self.response_val - - self.db._request = _request - self.db._request_json = _request_json - - def test__sanity_same_signature(self): - my_request_sig = inspect.getargspec(self.db._request) - my_request_sig = (['self'] + my_request_sig[0],) + my_request_sig[1:] - self.assertEqual( - my_request_sig, - inspect.getargspec(http_database.HTTPDatabase._request)) - my_request_json_sig = inspect.getargspec(self.db._request_json) - my_request_json_sig = ((['self'] + my_request_json_sig[0],) + - my_request_json_sig[1:]) - self.assertEqual( - my_request_json_sig, - inspect.getargspec(http_database.HTTPDatabase._request_json)) - - def test__ensure(self): - self.response_val = {'ok': True}, {} - self.db._ensure() - self.assertEqual(('PUT', [], {}, {}, None), self.got) - - def test__delete(self): - self.response_val = {'ok': True}, {} - self.db._delete() - self.assertEqual(('DELETE', [], {}, {}, None), self.got) - - def test__check(self): - self.response_val = {}, {} - res = self.db._check() - self.assertEqual({}, res) - self.assertEqual(('GET', [], None, None, None), self.got) - - def test_put_doc(self): - self.response_val = {'rev': 'doc-rev'}, {} - doc = Document('doc-id', None, '{"v": 1}') - res = self.db.put_doc(doc) - self.assertEqual('doc-rev', res) - self.assertEqual('doc-rev', doc.rev) - self.assertEqual(('PUT', ['doc', 'doc-id'], {}, - '{"v": 1}', 'application/json'), self.got) - - self.response_val = {'rev': 'doc-rev-2'}, {} - doc.content = {"v": 2} - res = self.db.put_doc(doc) - self.assertEqual('doc-rev-2', res) - self.assertEqual('doc-rev-2', doc.rev) - self.assertEqual(('PUT', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, - '{"v": 2}', 'application/json'), self.got) - - def test_get_doc(self): - self.response_val = '{"v": 2}', {'x-u1db-rev': 'doc-rev', - 'x-u1db-has-conflicts': 'false'} - self.assertGetDoc(self.db, 'doc-id', 'doc-rev', '{"v": 2}', False) - self.assertEqual( - ('GET', ['doc', 'doc-id'], {'include_deleted': False}, None, None), - self.got) - - def test_get_doc_non_existing(self): - self.response_val = errors.DocumentDoesNotExist() - self.assertIs(None, self.db.get_doc('not-there')) - self.assertEqual( - ('GET', ['doc', 'not-there'], {'include_deleted': False}, None, - None), self.got) - - def test_get_doc_deleted(self): - self.response_val = errors.DocumentDoesNotExist() - self.assertIs(None, self.db.get_doc('deleted')) - self.assertEqual( - ('GET', ['doc', 'deleted'], {'include_deleted': False}, None, - None), self.got) - - def test_get_doc_deleted_include_deleted(self): - self.response_val = errors.HTTPError( - 404, - json.dumps({"error": errors.DOCUMENT_DELETED}), - {'x-u1db-rev': 'doc-rev-gone', - 'x-u1db-has-conflicts': 'false'}) - doc = self.db.get_doc('deleted', include_deleted=True) - self.assertEqual('deleted', doc.doc_id) - self.assertEqual('doc-rev-gone', doc.rev) - self.assertIs(None, doc.content) - self.assertEqual( - ('GET', ['doc', 'deleted'], {'include_deleted': True}, None, None), - self.got) - - def test_get_doc_pass_through_errors(self): - self.response_val = errors.HTTPError(500, 'Crash.') - self.assertRaises(errors.HTTPError, - self.db.get_doc, 'something-something') - - def test_create_doc_with_id(self): - self.response_val = {'rev': 'doc-rev'}, {} - new_doc = self.db.create_doc_from_json('{"v": 1}', doc_id='doc-id') - self.assertEqual('doc-rev', new_doc.rev) - self.assertEqual('doc-id', new_doc.doc_id) - self.assertEqual('{"v": 1}', new_doc.get_json()) - self.assertEqual(('PUT', ['doc', 'doc-id'], {}, - '{"v": 1}', 'application/json'), self.got) - - def test_create_doc_without_id(self): - self.response_val = {'rev': 'doc-rev-2'}, {} - new_doc = self.db.create_doc_from_json('{"v": 3}') - self.assertEqual('D-', new_doc.doc_id[:2]) - self.assertEqual('doc-rev-2', new_doc.rev) - self.assertEqual('{"v": 3}', new_doc.get_json()) - self.assertEqual(('PUT', ['doc', new_doc.doc_id], {}, - '{"v": 3}', 'application/json'), self.got) - - def test_delete_doc(self): - self.response_val = {'rev': 'doc-rev-gone'}, {} - doc = Document('doc-id', 'doc-rev', None) - self.db.delete_doc(doc) - self.assertEqual('doc-rev-gone', doc.rev) - self.assertEqual(('DELETE', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, - None, None), self.got) - - def test_get_sync_target(self): - st = self.db.get_sync_target() - self.assertIsInstance(st, http_target.HTTPSyncTarget) - self.assertEqual(st._url, self.db._url) - - def test_get_sync_target_inherits_oauth_credentials(self): - self.db.set_oauth_credentials(tests.consumer1.key, - tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - st = self.db.get_sync_target() - self.assertEqual(self.db._creds, st._creds) - - -@skip("Skiping tests imported from U1DB.") -class TestHTTPDatabaseCtrWithCreds(tests.TestCase): - - def test_ctr_with_creds(self): - db1 = http_database.HTTPDatabase('http://dbs/db', creds={'oauth': { - 'consumer_key': tests.consumer1.key, - 'consumer_secret': tests.consumer1.secret, - 'token_key': tests.token1.key, - 'token_secret': tests.token1.secret - }}) - self.assertIn('oauth', db1._creds) - - -@skip("Skiping tests imported from U1DB.") -class TestHTTPDatabaseIntegration(tests.TestCaseWithServer): - - make_app_with_state = staticmethod(make_http_app) - - def setUp(self): - super(TestHTTPDatabaseIntegration, self).setUp() - self.startServer() - - def test_non_existing_db(self): - db = http_database.HTTPDatabase(self.getURL('not-there')) - self.assertRaises(errors.DatabaseDoesNotExist, db.get_doc, 'doc1') - - def test__ensure(self): - db = http_database.HTTPDatabase(self.getURL('new')) - db._ensure() - self.assertIs(None, db.get_doc('doc1')) - - def test__delete(self): - self.request_state._create_database('db0') - db = http_database.HTTPDatabase(self.getURL('db0')) - db._delete() - self.assertRaises(errors.DatabaseDoesNotExist, - self.request_state.check_database, 'db0') - - def test_open_database_existing(self): - self.request_state._create_database('db0') - db = http_database.HTTPDatabase.open_database(self.getURL('db0'), - create=False) - self.assertIs(None, db.get_doc('doc1')) - - def test_open_database_non_existing(self): - self.assertRaises(errors.DatabaseDoesNotExist, - http_database.HTTPDatabase.open_database, - self.getURL('not-there'), - create=False) - - def test_open_database_create(self): - db = http_database.HTTPDatabase.open_database(self.getURL('new'), - create=True) - self.assertIs(None, db.get_doc('doc1')) - - def test_delete_database_existing(self): - self.request_state._create_database('db0') - http_database.HTTPDatabase.delete_database(self.getURL('db0')) - self.assertRaises(errors.DatabaseDoesNotExist, - self.request_state.check_database, 'db0') - - def test_doc_ids_needing_quoting(self): - db0 = self.request_state._create_database('db0') - db = http_database.HTTPDatabase.open_database(self.getURL('db0'), - create=False) - doc = Document('%fff', None, '{}') - db.put_doc(doc) - self.assertGetDoc(db0, '%fff', doc.rev, '{}', False) - self.assertGetDoc(db, '%fff', doc.rev, '{}', False) diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py deleted file mode 100644 index e177a808..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Test support for client-side https support.""" - -import os -import ssl -import sys - -from paste import httpserver -from unittest import skip - -from u1db.remote import http_client -from u1db.remote import http_target - -from leap import soledad -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import make_oauth_http_app - - -def https_server_def(): - def make_server(host_port, application): - from OpenSSL import SSL - cert_file = os.path.join(os.path.dirname(__file__), 'testing-certs', - 'testing.cert') - key_file = os.path.join(os.path.dirname(__file__), 'testing-certs', - 'testing.key') - ssl_context = SSL.Context(SSL.SSLv23_METHOD) - ssl_context.use_privatekey_file(key_file) - ssl_context.use_certificate_chain_file(cert_file) - srv = httpserver.WSGIServerBase(application, host_port, - httpserver.WSGIHandler, - ssl_context=ssl_context - ) - - def shutdown_request(req): - req.shutdown() - srv.close_request(req) - - srv.shutdown_request = shutdown_request - application.base_url = "https://localhost:%s" % srv.server_address[1] - return srv - return make_server, "shutdown", "https" - - -def oauth_https_sync_target(test, host, path): - _, port = test.server.server_address - st = http_target.HTTPSyncTarget('https://%s:%d/~/%s' % (host, port, path)) - st.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - return st - - -@skip("Skiping tests imported from U1DB.") -class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): - - scenarios = [ - ('oauth_https', {'server_def': https_server_def, - 'make_app_with_state': make_oauth_http_app, - 'make_document_for_test': - tests.make_document_for_test, - 'sync_target': oauth_https_sync_target - }), - ] - - def setUp(self): - try: - import OpenSSL # noqa - except ImportError: - self.skipTest("Requires pyOpenSSL") - self.cacert_pem = os.path.join(os.path.dirname(__file__), - 'testing-certs', 'cacert.pem') - # The default u1db http_client class for doing HTTPS only does HTTPS - # if the platform is linux. Because of this, soledad replaces that - # class with one that will do HTTPS independent of the platform. In - # order to maintain the compatibility with u1db default tests, we undo - # that replacement here. - http_client._VerifiedHTTPSConnection = \ - soledad.client.api.old__VerifiedHTTPSConnection - super(TestHttpSyncTargetHttpsSupport, self).setUp() - - def getSyncTarget(self, host, path=None, cert_file=None): - if self.server is None: - self.startServer() - return self.sync_target(self, host, path, cert_file=cert_file) - - def test_working(self): - self.startServer() - db = self.request_state._create_database('test') - self.patch(http_client, 'CA_CERTS', self.cacert_pem) - remote_target = self.getSyncTarget('localhost', 'test') - remote_target.record_sync_info('other-id', 2, 'T-id') - self.assertEqual( - (2, 'T-id'), db._get_replica_gen_and_trans_id('other-id')) - - def test_cannot_verify_cert(self): - if not sys.platform.startswith('linux'): - self.skipTest( - "XXX certificate verification happens on linux only for now") - self.startServer() - # don't print expected traceback server-side - self.server.handle_error = lambda req, cli_addr: None - self.request_state._create_database('test') - remote_target = self.getSyncTarget('localhost', 'test') - try: - remote_target.record_sync_info('other-id', 2, 'T-id') - except ssl.SSLError, e: - self.assertIn("certificate verify failed", str(e)) - else: - self.fail("certificate verification should have failed.") - - def test_host_mismatch(self): - if not sys.platform.startswith('linux'): - self.skipTest( - "XXX certificate verification happens on linux only for now") - self.startServer() - self.request_state._create_database('test') - self.patch(http_client, 'CA_CERTS', self.cacert_pem) - remote_target = self.getSyncTarget('127.0.0.1', 'test') - self.assertRaises( - http_client.CertificateError, remote_target.record_sync_info, - 'other-id', 2, 'T-id') - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py deleted file mode 100644 index ee249e6e..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see <http://www.gnu.org/licenses/>. - -"""Test u1db.open""" - -import os - -from u1db import ( - errors, - open as u1db_open, -) -from unittest import skip -from leap.soledad.common.tests import u1db_tests as tests -from u1db.backends import sqlite_backend -from leap.soledad.common.tests.u1db_tests.test_backends \ - import TestAlternativeDocument - - -@skip("Skiping tests imported from U1DB.") -class TestU1DBOpen(tests.TestCase): - - def setUp(self): - super(TestU1DBOpen, self).setUp() - tmpdir = self.createTempDir() - self.db_path = tmpdir + '/test.db' - - def test_open_no_create(self): - self.assertRaises(errors.DatabaseDoesNotExist, - u1db_open, self.db_path, create=False) - self.assertFalse(os.path.exists(self.db_path)) - - def test_open_create(self): - db = u1db_open(self.db_path, create=True) - self.addCleanup(db.close) - self.assertTrue(os.path.exists(self.db_path)) - self.assertIsInstance(db, sqlite_backend.SQLiteDatabase) - - def test_open_with_factory(self): - db = u1db_open(self.db_path, create=True, - document_factory=TestAlternativeDocument) - self.addCleanup(db.close) - self.assertEqual(TestAlternativeDocument, db._factory) - - def test_open_existing(self): - db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) - self.addCleanup(db.close) - doc = db.create_doc_from_json(tests.simple_doc) - # Even though create=True, we shouldn't wipe the db - db2 = u1db_open(self.db_path, create=True) - self.addCleanup(db2.close) - doc2 = db2.get_doc(doc.doc_id) - self.assertEqual(doc, doc2) - - def test_open_existing_no_create(self): - db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) - self.addCleanup(db.close) - db2 = u1db_open(self.db_path, create=False) - self.addCleanup(db2.close) - self.assertIsInstance(db2, sqlite_backend.SQLitePartialExpandDatabase) diff --git a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile deleted file mode 100644 index 2385e75b..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -CATOP=./demoCA -ORIG_CONF=/usr/lib/ssl/openssl.cnf -ELEVEN_YEARS=-days 4015 - -init: - cp $(ORIG_CONF) ca.conf - install -d $(CATOP) - install -d $(CATOP)/certs - install -d $(CATOP)/crl - install -d $(CATOP)/newcerts - install -d $(CATOP)/private - touch $(CATOP)/index.txt - echo 01>$(CATOP)/crlnumber - @echo '**** Making CA certificate ...' - openssl req -nodes -new \ - -newkey rsa -keyout $(CATOP)/private/cakey.pem \ - -out $(CATOP)/careq.pem \ - -multivalue-rdn \ - -subj "/C=UK/ST=-/O=u1db LOCAL TESTING ONLY, DO NO TRUST/CN=u1db testing CA" - openssl ca -config ./ca.conf -create_serial \ - -out $(CATOP)/cacert.pem $(ELEVEN_YEARS) -batch \ - -keyfile $(CATOP)/private/cakey.pem -selfsign \ - -extensions v3_ca -infiles $(CATOP)/careq.pem - -pems: - cp ./demoCA/cacert.pem . - openssl req -new -config ca.conf \ - -multivalue-rdn \ - -subj "/O=u1db LOCAL TESTING ONLY, DO NOT TRUST/CN=localhost" \ - -nodes -keyout testing.key -out newreq.pem $(ELEVEN_YEARS) - openssl ca -batch -config ./ca.conf $(ELEVEN_YEARS) \ - -policy policy_anything \ - -out testing.cert -infiles newreq.pem - -.PHONY: init pems diff --git a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem deleted file mode 100644 index c019a730..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem +++ /dev/null @@ -1,58 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - e4:de:01:76:c4:78:78:7e - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA - Validity - Not Before: May 3 11:11:11 2012 GMT - Not After : May 1 11:11:11 2023 GMT - Subject: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (1024 bit) - Modulus: - 00:bc:91:a5:7f:7d:37:f7:06:c7:db:5b:83:6a:6b: - 63:c3:8b:5c:f7:84:4d:97:6d:d4:be:bf:e7:79:a8: - c1:03:57:ec:90:d4:20:e7:02:95:d9:a6:49:e3:f9: - 9a:ea:37:b9:b2:02:62:ab:40:d3:42:bb:4a:4e:a2: - 47:71:0f:1d:a2:c5:94:a1:cf:35:d3:23:32:42:c0: - 1e:8d:cb:08:58:fb:8a:5c:3e:ea:eb:d5:2c:ed:d6: - aa:09:b4:b5:7d:e3:45:c9:ae:c2:82:b2:ae:c0:81: - bc:24:06:65:a9:e7:e0:61:ac:25:ee:53:d3:d7:be: - 22:f7:00:a2:ad:c6:0e:3a:39 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Key Identifier: - DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D - X509v3 Authority Key Identifier: - keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D - - X509v3 Basic Constraints: - CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 72:9b:c1:f7:07:65:83:36:25:4e:01:2f:b7:4a:f2:a4:00:28: - 80:c7:56:2c:32:39:90:13:61:4b:bb:12:c5:44:9d:42:57:85: - 28:19:70:69:e1:43:c8:bd:11:f6:94:df:91:2d:c3:ea:82:8d: - b4:8f:5d:47:a3:00:99:53:29:93:27:6c:c5:da:c1:20:6f:ab: - ec:4a:be:34:f3:8f:02:e5:0c:c0:03:ac:2b:33:41:71:4f:0a: - 72:5a:b4:26:1a:7f:81:bc:c0:95:8a:06:87:a8:11:9f:5c:73: - 38:df:5a:69:40:21:29:ad:46:23:56:75:e1:e9:8b:10:18:4c: - 7b:54 ------BEGIN CERTIFICATE----- -MIICkjCCAfugAwIBAgIJAOTeAXbEeHh+MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV -BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg -T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x -MjA1MDMxMTExMTFaFw0yMzA1MDExMTExMTFaMGIxCzAJBgNVBAYTAlVLMQowCAYD -VQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcgT05MWSwgRE8gTk8g -VFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTCBnzANBgkqhkiG9w0BAQEF -AAOBjQAwgYkCgYEAvJGlf3039wbH21uDamtjw4tc94RNl23Uvr/neajBA1fskNQg -5wKV2aZJ4/ma6je5sgJiq0DTQrtKTqJHcQ8dosWUoc810yMyQsAejcsIWPuKXD7q -69Us7daqCbS1feNFya7CgrKuwIG8JAZlqefgYawl7lPT174i9wCircYOOjkCAwEA -AaNQME4wHQYDVR0OBBYEFNs9k1FsMhVUjxBQ/ElPNhUou5VtMB8GA1UdIwQYMBaA -FNs9k1FsMhVUjxBQ/ElPNhUou5VtMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEF -BQADgYEAcpvB9wdlgzYlTgEvt0rypAAogMdWLDI5kBNhS7sSxUSdQleFKBlwaeFD -yL0R9pTfkS3D6oKNtI9dR6MAmVMpkydsxdrBIG+r7Eq+NPOPAuUMwAOsKzNBcU8K -clq0Jhp/gbzAlYoGh6gRn1xzON9aaUAhKa1GI1Z14emLEBhMe1Q= ------END CERTIFICATE----- diff --git a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert deleted file mode 100644 index 985684fb..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert +++ /dev/null @@ -1,61 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - e4:de:01:76:c4:78:78:7f - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA - Validity - Not Before: May 3 11:11:14 2012 GMT - Not After : May 1 11:11:14 2023 GMT - Subject: O=u1db LOCAL TESTING ONLY, DO NOT TRUST, CN=localhost - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (1024 bit) - Modulus: - 00:c6:1d:72:d3:c5:e4:fc:d1:4c:d9:e4:08:3e:90: - 10:ce:3f:1f:87:4a:1d:4f:7f:2a:5a:52:c9:65:4f: - d9:2c:bf:69:75:18:1a:b5:c9:09:32:00:47:f5:60: - aa:c6:dd:3a:87:37:5f:16:be:de:29:b5:ea:fc:41: - 7e:eb:77:bb:df:63:c3:06:1e:ed:e9:a0:67:1a:f1: - ec:e1:9d:f7:9c:8f:1c:fa:c3:66:7b:39:dc:70:ae: - 09:1b:9c:c0:9a:c4:90:77:45:8e:39:95:a9:2f:92: - 43:bd:27:07:5a:99:51:6e:76:a0:af:dd:b1:2c:8f: - ca:8b:8c:47:0d:f6:6e:fc:69 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Basic Constraints: - CA:FALSE - Netscape Comment: - OpenSSL Generated Certificate - X509v3 Subject Key Identifier: - 1C:63:85:E1:1D:F3:89:2E:6C:4E:3F:FB:D0:10:64:5A:C1:22:6A:2A - X509v3 Authority Key Identifier: - keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D - - Signature Algorithm: sha1WithRSAEncryption - 1d:6d:3e:bd:93:fd:bd:3e:17:b8:9f:f0:99:7f:db:50:5c:b2: - 01:42:03:b5:d5:94:05:d3:f6:8e:80:82:55:47:1f:58:f2:18: - 6c:ab:ef:43:2c:2f:10:e1:7c:c4:5c:cc:ac:50:50:22:42:aa: - 35:33:f5:b9:f3:a6:66:55:d9:36:f4:f2:e4:d4:d9:b5:2c:52: - 66:d4:21:17:97:22:b8:9b:d7:0e:7c:3d:ce:85:19:ca:c4:d2: - 58:62:31:c6:18:3e:44:fc:f4:30:b6:95:87:ee:21:4a:08:f0: - af:3c:8f:c4:ba:5e:a1:5c:37:1a:7d:7b:fe:66:ae:62:50:17: - 31:ca ------BEGIN CERTIFICATE----- -MIICnzCCAgigAwIBAgIJAOTeAXbEeHh/MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV -BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg -T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x -MjA1MDMxMTExMTRaFw0yMzA1MDExMTExMTRaMEQxLjAsBgNVBAoMJXUxZGIgTE9D -QUwgVEVTVElORyBPTkxZLCBETyBOT1QgVFJVU1QxEjAQBgNVBAMMCWxvY2FsaG9z -dDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxh1y08Xk/NFM2eQIPpAQzj8f -h0odT38qWlLJZU/ZLL9pdRgatckJMgBH9WCqxt06hzdfFr7eKbXq/EF+63e732PD -Bh7t6aBnGvHs4Z33nI8c+sNmeznccK4JG5zAmsSQd0WOOZWpL5JDvScHWplRbnag -r92xLI/Ki4xHDfZu/GkCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0E -HxYdT3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBxjheEd -84kubE4/+9AQZFrBImoqMB8GA1UdIwQYMBaAFNs9k1FsMhVUjxBQ/ElPNhUou5Vt -MA0GCSqGSIb3DQEBBQUAA4GBAB1tPr2T/b0+F7if8Jl/21BcsgFCA7XVlAXT9o6A -glVHH1jyGGyr70MsLxDhfMRczKxQUCJCqjUz9bnzpmZV2Tb08uTU2bUsUmbUIReX -Irib1w58Pc6FGcrE0lhiMcYYPkT89DC2lYfuIUoI8K88j8S6XqFcNxp9e/5mrmJQ -FzHK ------END CERTIFICATE----- diff --git a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key deleted file mode 100644 index d83d4920..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMYdctPF5PzRTNnk -CD6QEM4/H4dKHU9/KlpSyWVP2Sy/aXUYGrXJCTIAR/VgqsbdOoc3Xxa+3im16vxB -fut3u99jwwYe7emgZxrx7OGd95yPHPrDZns53HCuCRucwJrEkHdFjjmVqS+SQ70n -B1qZUW52oK/dsSyPyouMRw32bvxpAgMBAAECgYBs3lXxhjg1rhabTjIxnx19GTcM -M3Az9V+izweZQu3HJ1CeZiaXauhAr+LbNsniCkRVddotN6oCJdQB10QVxXBZc9Jz -HPJ4zxtZfRZlNMTMmG7eLWrfxpgWnb/BUjDb40yy1nhr9yhDUnI/8RoHDRHnAEHZ -/CnHGUrqcVcrY5zJAQJBAPLhBJg9W88JVmcOKdWxRgs7dLHnZb999Kv1V5mczmAi -jvGvbUmucqOqke6pTUHNYyNHqU6pySzGUi2cH+BAkFECQQDQ0VoAOysg6FVoT15v -tGh57t5sTiCZZ7PS8jwvtThsgA+vcf6c16XWzXgjGXSap4r2QDOY2rI5lsWLaQ8T -+fyZAkAfyFJRmbXp4c7srW3MCOahkaYzoZQu+syJtBFCiMJ40gzik5I5khpuUGPI -V19EvRu8AiSlppIsycb3MPb64XgBAkEAy7DrUf5le5wmc7G4NM6OeyJ+5LbxJbL6 -vnJ8My1a9LuWkVVpQCU7J+UVo2dZTuLPspW9vwTVhUeFOxAoHRxlQQJAFem93f7m -el2BkB2EFqU3onPejkZ5UrDmfmeOQR1axMQNSXqSxcJxqa16Ru1BWV2gcWRbwajQ -oc+kuJThu/r/Ug== ------END PRIVATE KEY----- diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py deleted file mode 100644 index d4510686..00000000 --- a/common/src/leap/soledad/common/tests/util.py +++ /dev/null @@ -1,423 +0,0 @@ -# -*- coding: utf-8 -*- -# util.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -Utilities used by multiple test suites. -""" - - -import os -import tempfile -import shutil -import random -import string -import u1db -import couchdb - -from uuid import uuid4 -from mock import Mock -from urlparse import urljoin -from StringIO import StringIO -from pysqlcipher import dbapi2 - -from u1db import sync -from u1db.remote import http_database - -from twisted.trial import unittest - -from leap.common.testing.basetest import BaseLeapTest - -from leap.soledad.common import soledad_assert -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.couch.state import CouchServerState - -from leap.soledad.common.crypto import ENC_SCHEME_KEY - -from leap.soledad.client import Soledad -from leap.soledad.client import http_target -from leap.soledad.client import auth -from leap.soledad.client.crypto import decrypt_doc_dict -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.sqlcipher import SQLCipherOptions - -from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -PASSWORD = '123456' -ADDRESS = 'leap@leap.se' - - -def make_local_db_and_target(test): - db = test.create_database('test') - st = db.get_sync_target() - return db, st - - -def make_sqlcipher_database_for_test(test, replica_uid): - db = SQLCipherDatabase( - SQLCipherOptions(':memory:', PASSWORD)) - db._set_replica_uid(replica_uid) - return db - - -def copy_sqlcipher_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - new_db = make_sqlcipher_database_for_test(test, None) - tmpfile = StringIO() - for line in db._db_handle.iterdump(): - if 'sqlite_sequence' not in line: # work around bug in iterdump - tmpfile.write('%s\n' % line) - tmpfile.seek(0) - new_db._db_handle = dbapi2.connect(':memory:') - new_db._db_handle.cursor().executescript(tmpfile.read()) - new_db._db_handle.commit() - new_db._set_replica_uid(db._replica_uid) - new_db._factory = db._factory - return new_db - - -def make_soledad_app(state): - return SoledadApp(state) - - -def make_token_soledad_app(state): - app = SoledadApp(state) - - def _verify_authentication_data(uuid, auth_data): - if uuid.startswith('user-') and auth_data == 'auth-token': - return True - return False - - # we test for action authorization in leap.soledad.common.tests.test_server - def _verify_authorization(uuid, environ): - return True - - application = SoledadTokenAuthMiddleware(app) - application._verify_authentication_data = _verify_authentication_data - application._verify_authorization = _verify_authorization - return application - - -def make_soledad_document_for_test(test, doc_id, rev, content, - has_conflicts=False): - return SoledadDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -def make_token_http_database_for_test(test, replica_uid): - test.startServer() - test.request_state._create_database(replica_uid) - - class _HTTPDatabaseWithToken( - http_database.HTTPDatabase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - http_db = _HTTPDatabaseWithToken(test.getURL('test')) - http_db.set_token_credentials('user-uuid', 'auth-token') - return http_db - - -def copy_token_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') - return http_db - - -def sync_via_synchronizer(test, db_source, db_target, trace_hook=None, - trace_hook_shallow=None): - target = db_target.get_sync_target() - trace_hook = trace_hook or trace_hook_shallow - if trace_hook: - target._set_trace_hook(trace_hook) - return sync.Synchronizer(db_source, target).sync() - - -class MockedSharedDBTest(object): - - def get_default_shared_mock(self, put_doc_side_effect=None, - get_doc_return_value=None): - """ - Get a default class for mocking the shared DB - """ - class defaultMockSharedDB(object): - get_doc = Mock(return_value=get_doc_return_value) - put_doc = Mock(side_effect=put_doc_side_effect) - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) - open = Mock(return_value=None) - close = Mock(return_value=None) - syncable = True - - def __call__(self): - return self - return defaultMockSharedDB - - -def soledad_sync_target( - test, path, source_replica_uid=uuid4().hex, - sync_db=None, sync_enc_pool=None): - creds = {'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }} - return http_target.SoledadHTTPSyncTarget( - test.getURL(path), - source_replica_uid, - creds, - test._soledad._crypto, - None, # cert_file - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - - -# redefine the base leap test class so it inherits from twisted trial's -# TestCase. This is needed so trial knows that it has to manage a reactor and -# wait for deferreds returned by tests to be fired. -BaseLeapTest = type( - 'BaseLeapTest', (unittest.TestCase,), dict(BaseLeapTest.__dict__)) - - -class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest): - - """ - Instantiates Soledad for usage in tests. - """ - defer_sync_encryption = False - - def setUp(self): - # The following snippet comes from BaseLeapTest.setUpClass, but we - # repeat it here because twisted.trial does not work with - # setUpClass/tearDownClass. - - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - self.db2_file = os.path.join(self.tempdir, "db2.u1db") - self.email = ADDRESS - # open test dbs - self._db1 = u1db.open(self.db1_file, create=True, - document_factory=SoledadDocument) - self._db2 = u1db.open(self.db2_file, create=True, - document_factory=SoledadDocument) - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - - # initialize soledad by hand so we can control keys - # XXX check if this soledad is actually used - self._soledad = self._soledad_instance( - prefix=self.rand_prefix, user=self.email) - - def tearDown(self): - self._db1.close() - self._db2.close() - self._soledad.close() - - # restore paths - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - - def _delete_temporary_dirs(): - # XXX should not access "private" attrs - for f in [self._soledad.local_db_path, - self._soledad.secrets.secrets_path]: - if os.path.isfile(f): - os.unlink(f) - # The following snippet comes from BaseLeapTest.setUpClass, but we - # repeat it here because twisted.trial does not work with - # setUpClass/tearDownClass. - soledad_assert( - self.tempdir.startswith('/tmp/leap_tests-'), - "beware! tried to remove a dir which does not " - "live in temporal folder!") - shutil.rmtree(self.tempdir) - - from twisted.internet import reactor - reactor.addSystemEventTrigger( - "after", "shutdown", _delete_temporary_dirs) - - def _soledad_instance(self, user=ADDRESS, passphrase=u'123', - prefix='', - secrets_path='secrets.json', - local_db_path='soledad.u1db', - server_url='https://127.0.0.1/', - cert_file=None, - shared_db_class=None, - auth_token='auth-token', - userid=ADDRESS): - - def _put_doc_side_effect(doc): - self._doc_put = doc - - if shared_db_class is not None: - MockSharedDB = shared_db_class - else: - MockSharedDB = self.get_default_shared_mock( - _put_doc_side_effect) - - soledad = Soledad( - user, - passphrase, - secrets_path=os.path.join( - self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, # Soledad will fail if not given an url. - cert_file=cert_file, - defer_encryption=self.defer_sync_encryption, - shared_db=MockSharedDB(), - auth_token=auth_token, - userid=userid) - self.addCleanup(soledad.close) - return soledad - - def assertGetEncryptedDoc( - self, db, doc_id, doc_rev, content, has_conflicts): - """ - Assert that the document in the database looks correct. - """ - exp_doc = self.make_document(doc_id, doc_rev, content, - has_conflicts=has_conflicts) - doc = db.get_doc(doc_id) - - if ENC_SCHEME_KEY in doc.content: - # XXX check for SYM_KEY too - key = self._soledad._crypto.doc_passphrase(doc.doc_id) - secret = self._soledad._crypto.secret - decrypted = decrypt_doc_dict( - doc.content, doc.doc_id, doc.rev, - key, secret) - doc.set_json(decrypted) - self.assertEqual(exp_doc.doc_id, doc.doc_id) - self.assertEqual(exp_doc.rev, doc.rev) - self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts) - self.assertEqual(exp_doc.content, doc.content) - - -class CouchDBTestCase(unittest.TestCase, MockedSharedDBTest): - - """ - TestCase base class for tests against a real CouchDB server. - """ - - def setUp(self): - """ - Make sure we have a CouchDB instance for a test. - """ - self.couch_port = 5984 - self.couch_url = 'http://localhost:%d' % self.couch_port - self.couch_server = couchdb.Server(self.couch_url) - - def delete_db(self, name): - try: - self.couch_server.delete(name) - except: - # ignore if already missing - pass - - -class CouchServerStateForTests(CouchServerState): - - """ - This is a slightly modified CouchDB server state that allows for creating - a database. - - Ordinarily, the CouchDB server state does not allow some operations, - because for security purposes the Soledad Server should not even have - enough permissions to perform them. For tests, we allow database creation, - otherwise we'd have to create those databases in setUp/tearDown methods, - which is less pleasant than allowing the db to be automatically created. - """ - - def __init__(self, *args, **kwargs): - self.dbs = [] - super(CouchServerStateForTests, self).__init__(*args, **kwargs) - - def _create_database(self, replica_uid=None, dbname=None): - """ - Create db and append to a list, allowing test to close it later - """ - dbname = dbname or ('test-%s' % uuid4().hex) - db = CouchDatabase.open_database( - urljoin(self.couch_url, dbname), - True, - replica_uid=replica_uid or 'test', - ensure_ddocs=True) - self.dbs.append(db) - return db - - def ensure_database(self, dbname): - db = self._create_database(dbname=dbname) - return db, db.replica_uid - - -class SoledadWithCouchServerMixin( - BaseSoledadTest, - CouchDBTestCase): - - def setUp(self): - CouchDBTestCase.setUp(self) - BaseSoledadTest.setUp(self) - main_test_class = getattr(self, 'main_test_class', None) - if main_test_class is not None: - main_test_class.setUp(self) - - def tearDown(self): - main_test_class = getattr(self, 'main_test_class', None) - if main_test_class is not None: - main_test_class.tearDown(self) - # delete the test database - BaseSoledadTest.tearDown(self) - CouchDBTestCase.tearDown(self) - - def make_app(self): - self.request_state = CouchServerStateForTests(self.couch_url) - self.addCleanup(self.delete_dbs) - return self.make_app_with_state(self.request_state) - - def delete_dbs(self): - for db in self.request_state.dbs: - self.delete_db(db._dbname) |